diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index 6a9527af375..4fafa4a8c58 100644
--- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -59,6 +59,7 @@
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class Credentials implements Writable {
+
public enum SerializedFormat {
WRITABLE((byte) 0x00),
PROTOBUF((byte) 0x01);
diff --git hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
index 898c94ec33b..1870b228093 100644
--- hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
+++ hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.test.GenericTestUtils;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
@@ -50,6 +51,7 @@
import java.security.SignatureException;
import java.security.cert.CertificateEncodingException;
import javax.security.auth.x500.X500Principal;
+
import org.bouncycastle.x509.X509V1CertificateGenerator;
public class KeyStoreTestUtil {
@@ -127,9 +129,16 @@ public static void createKeyStore(String filename,
String password, String alias,
Key privateKey, Certificate cert)
throws GeneralSecurityException, IOException {
- KeyStore ks = createEmptyKeyStore();
- ks.setKeyEntry(alias, privateKey, password.toCharArray(),
+ createKeyStore(filename, password, alias, privateKey,
new Certificate[]{cert});
+ }
+
+ public static void createKeyStore(String filename,
+ String password, String alias,
+ Key privateKey, Certificate[] certs)
+ throws GeneralSecurityException, IOException {
+ KeyStore ks = createEmptyKeyStore();
+ ks.setKeyEntry(alias, privateKey, password.toCharArray(), certs);
saveKeyStore(ks, filename, password);
}
@@ -174,6 +183,14 @@ public static void createTrustStore(String filename,
saveKeyStore(ks, filename, password);
}
+ public static KeyStore bytesToKeyStore(byte[] bytes, String password)
+ throws GeneralSecurityException, IOException {
+ KeyStore keyStore = createEmptyKeyStore();
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ keyStore.load(bais, password.toCharArray());
+ return keyStore;
+ }
+
public static void cleanupSSLConfig(String keystoresDir, String sslConfDir)
throws Exception {
File f = new File(keystoresDir + "/clientKS.jks");
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index b63fe614130..106a496d603 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -47,6 +47,31 @@
public static final String CONTAINER_TOKEN_FILE_ENV_NAME =
UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
+ /**
+ * The file into which the keystore containing the AM's certificate is written
+ */
+ public static final String KEYSTORE_FILE_LOCATION_ENV_NAME =
+ "KEYSTORE_FILE_LOCATION";
+
+ /**
+ * The password for the AM's keystore
+ */
+ public static final String KEYSTORE_PASSWORD_ENV_NAME =
+ "KEYSTORE_PASSWORD";
+
+ /**
+ * The file into which the truststore containing the AM's certificate is
+ * written
+ */
+ public static final String TRUSTSTORE_FILE_LOCATION_ENV_NAME =
+ "TRUSTSTORE_FILE_LOCATION";
+
+ /**
+ * The password for the AM's truststore
+ */
+ public static final String TRUSTSTORE_PASSWORD_ENV_NAME =
+ "TRUSTSTORE_PASSWORD";
+
/**
* The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
* ApplicationMaster's environment only. This states that for all non-relative
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6488ebfc4ec..fd7daa800f9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2131,6 +2131,26 @@ public static boolean isAclEnabled(Configuration conf) {
public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS =
3000;
+ /**
+ * Specifies what the RM does regarding HTTPS enforcement for communication
+ * with AM Web Servers, as well as generating and providing certificates.
+ * Possible values are:
+ *
+ *
NONE - the RM will do nothing special.
+ *
LENIENT - the RM will generate and provide a keystore and truststore
+ * to the AM, which it is free to use for HTTPS in its tracking URL web
+ * server. The RM proxy will still accept HTTP connections though for AMs
+ * that opt not to use HTTPS.
+ *
STRICT - this is the same as LENIENT, except that the RM proxy will
+ * only accept HTTPS connections to AMs; HTTP connections will be blocked
+ * and result in a warning page to the user.
+ *
+ */
+ public static final String RM_APPLICATION_HTTPS_POLICY =
+ RM_PREFIX + "application-https.policy";
+
+ public static final String DEFAULT_RM_APPLICATION_HTTPS_POLICY = "NONE";
+
/**
* Interval of time the linux container executor should try cleaning up
* cgroups entry when cleaning up a container. This is required due to what
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8e9f15be79c..9c33724ef7d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3503,6 +3503,24 @@
3000
+
+
+ Specifies what the RM does regarding HTTPS enforcement for communication
+ with AM Web Servers, as well as generating and providing certificates.
+ Possible values are:
+ - NONE - the RM will do nothing special.
+ - LENIENT - the RM will generate and provide a keystore and truststore
+ to the AM, which it is free to use for HTTPS in its tracking URL web
+ server. The RM proxy will still accept HTTP connections though for AMs
+ that opt not to use HTTPS.
+ - STRICT - this is the same as LENIENT, except that the RM proxy will
+ only accept HTTPS connections to AMs; HTTP connections will be blocked
+ and result in a warning page to the user.
+
+ yarn.resourcemanager.application-https.policy
+ NONE
+
+
Defines the limit of the diagnostics message of an application
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/AMSecretKeys.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/AMSecretKeys.java
new file mode 100644
index 00000000000..541a9ab80df
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/AMSecretKeys.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.security;
+
+import org.apache.hadoop.io.Text;
+
+public class AMSecretKeys {
+
+ public final static Text YARN_APPLICATION_AM_KEYSTORE =
+ new Text("yarn.application.am.keystore");
+ public final static Text YARN_APPLICATION_AM_KEYSTORE_PASSWORD =
+ new Text("yarn.application.am.keystore.password");
+ public final static Text YARN_APPLICATION_AM_TRUSTSTORE =
+ new Text("yarn.application.am.truststore");
+ public final static Text YARN_APPLICATION_AM_TRUSTSTORE_PASSWORD =
+ new Text("yarn.application.am.truststore.password");
+}
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 400133310fd..b552c1f93a2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -218,6 +218,8 @@ public int launchContainer(ContainerStartContext ctx)
Container container = ctx.getContainer();
Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
Path nmPrivateTokensPath = ctx.getNmPrivateTokensPath();
+ Path nmPrivateKeystorePath = ctx.getNmPrivateKeystorePath();
+ Path nmPrivateTruststorePath = ctx.getNmPrivateTruststorePath();
String user = ctx.getUser();
Path containerWorkDir = ctx.getContainerWorkDir();
List localDirs = ctx.getLocalDirs();
@@ -253,6 +255,18 @@ public int launchContainer(ContainerStartContext ctx)
new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE);
copyFile(nmPrivateTokensPath, tokenDst, user);
+ if (nmPrivateKeystorePath != null) {
+ Path keystoreDst =
+ new Path(containerWorkDir, ContainerLaunch.KEYSTORE_FILE);
+ copyFile(nmPrivateKeystorePath, keystoreDst, user);
+ }
+
+ if (nmPrivateTruststorePath != null) {
+ Path truststoreDst =
+ new Path(containerWorkDir, ContainerLaunch.TRUSTSTORE_FILE);
+ copyFile(nmPrivateTruststorePath, truststoreDst, user);
+ }
+
// copy launch script to work dir
Path launchDst =
new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 58d3068916d..3946eed5a04 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -658,6 +658,10 @@ private ContainerRuntimeContext buildContainerRuntimeContext(
ctx.getNmPrivateContainerScriptPath())
.setExecutionAttribute(NM_PRIVATE_TOKENS_PATH,
ctx.getNmPrivateTokensPath())
+ .setExecutionAttribute(NM_PRIVATE_KEYSTORE_PATH,
+ ctx.getNmPrivateKeystorePath())
+ .setExecutionAttribute(NM_PRIVATE_TRUSTSTORE_PATH,
+ ctx.getNmPrivateTruststorePath())
.setExecutionAttribute(PID_FILE_PATH, pidFilePath)
.setExecutionAttribute(LOCAL_DIRS, ctx.getLocalDirs())
.setExecutionAttribute(LOG_DIRS, ctx.getLogDirs())
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index f198e83e09a..008c09bed61 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -91,6 +91,7 @@
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
+import org.apache.hadoop.yarn.server.security.AMSecretKeys;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
@@ -112,6 +113,9 @@
public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";
+ public static final String KEYSTORE_FILE = "yarn_provided.keystore";
+ public static final String TRUSTSTORE_FILE = "yarn_provided.truststore";
+
private static final String PID_FILE_NAME_FMT = "%s.pid";
static final String EXIT_CODE_FILE_SUFFIX = ".exitcode";
@@ -232,6 +236,12 @@ public Integer call() {
getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
containerIdStr));
+ Path nmPrivateKeystorePath = dirsHandler.getLocalPathForWrite(
+ getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ + KEYSTORE_FILE);
+ Path nmPrivateTruststorePath = dirsHandler.getLocalPathForWrite(
+ getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ + TRUSTSTORE_FILE);
Path nmPrivateClasspathJarDir = dirsHandler.getLocalPathForWrite(
getContainerPrivateDir(appIdStr, containerIdStr));
@@ -267,6 +277,44 @@ public Integer call() {
appDirs.add(new Path(appsdir, appIdStr));
}
+ byte[] keystore = container.getCredentials().getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE);
+ if (keystore != null) {
+ try (DataOutputStream keystoreOutStream =
+ lfs.create(nmPrivateKeystorePath,
+ EnumSet.of(CREATE, OVERWRITE))) {
+ keystoreOutStream.write(keystore);
+ environment.put(ApplicationConstants.KEYSTORE_FILE_LOCATION_ENV_NAME,
+ new Path(containerWorkDir,
+ ContainerLaunch.KEYSTORE_FILE).toUri().getPath());
+ environment.put(ApplicationConstants.KEYSTORE_PASSWORD_ENV_NAME,
+ new String(container.getCredentials().getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE_PASSWORD),
+ StandardCharsets.UTF_8));
+ }
+ } else {
+ nmPrivateKeystorePath = null;
+ }
+ byte[] truststore = container.getCredentials().getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE);
+ if (truststore != null) {
+ try (DataOutputStream truststoreOutStream =
+ lfs.create(nmPrivateTruststorePath,
+ EnumSet.of(CREATE, OVERWRITE))) {
+ truststoreOutStream.write(truststore);
+ environment.put(
+ ApplicationConstants.TRUSTSTORE_FILE_LOCATION_ENV_NAME,
+ new Path(containerWorkDir,
+ ContainerLaunch.TRUSTSTORE_FILE).toUri().getPath());
+ environment.put(ApplicationConstants.TRUSTSTORE_PASSWORD_ENV_NAME,
+ new String(container.getCredentials().getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE_PASSWORD),
+ StandardCharsets.UTF_8));
+ }
+ } else {
+ nmPrivateTruststorePath = null;
+ }
+
// Set the token location too.
addToEnvMap(environment, nmEnvVars,
ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME,
@@ -304,6 +352,8 @@ public Integer call() {
.setLocalizedResources(localResources)
.setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath)
.setNmPrivateTokensPath(nmPrivateTokensPath)
+ .setNmPrivateKeystorePath(nmPrivateKeystorePath)
+ .setNmPrivateTruststorePath(nmPrivateTruststorePath)
.setUser(user)
.setAppId(appIdStr)
.setContainerWorkDir(containerWorkDir)
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
index a34ed625050..307ded5b6f9 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
@@ -34,6 +34,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
+import org.apache.hadoop.yarn.server.security.AMSecretKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -81,6 +82,12 @@ public Integer call() {
getNmPrivateContainerScriptPath(appIdStr, containerIdStr);
Path nmPrivateTokensPath =
getNmPrivateTokensPath(appIdStr, containerIdStr);
+ Path nmPrivateKeystorePath = (container.getCredentials().getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE) == null) ? null :
+ getNmPrivateKeystorePath(appIdStr, containerIdStr);
+ Path nmPrivateTruststorePath = (container.getCredentials().getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE) == null) ? null :
+ getNmPrivateTruststorePath(appIdStr, containerIdStr);
pidFilePath = getPidFilePath(appIdStr, containerIdStr);
LOG.info("Relaunch container with "
@@ -112,6 +119,8 @@ public Integer call() {
.setLocalizedResources(localResources)
.setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath)
.setNmPrivateTokensPath(nmPrivateTokensPath)
+ .setNmPrivateKeystorePath(nmPrivateKeystorePath)
+ .setNmPrivateTruststorePath(nmPrivateTruststorePath)
.setUser(container.getUser())
.setAppId(appIdStr)
.setContainerWorkDir(containerWorkDir)
@@ -173,6 +182,20 @@ private Path getNmPrivateTokensPath(String appIdStr,
containerIdStr));
}
+ private Path getNmPrivateKeystorePath(String appIdStr,
+ String containerIdStr) throws IOException {
+ return dirsHandler.getLocalPathForRead(
+ getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ + ContainerLaunch.KEYSTORE_FILE);
+ }
+
+ private Path getNmPrivateTruststorePath(String appIdStr,
+ String containerIdStr) throws IOException {
+ return dirsHandler.getLocalPathForRead(
+ getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+ + ContainerLaunch.TRUSTSTORE_FILE);
+ }
+
private Path getPidFilePath(String appIdStr,
String containerIdStr) throws IOException {
return dirsHandler.getLocalPathForRead(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 82ca6d91f24..b4cead2a21b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
@@ -106,8 +107,17 @@ public void launchContainer(ContainerRuntimeContext ctx)
ctx.getExecutionAttribute(CONTAINER_WORK_DIR).toString(),
ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH).toUri()
.getPath(),
- ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(),
- ctx.getExecutionAttribute(PID_FILE_PATH).toString(),
+ ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath());
+ Path keystorePath = ctx.getExecutionAttribute(NM_PRIVATE_KEYSTORE_PATH);
+ Path truststorePath = ctx.getExecutionAttribute(NM_PRIVATE_TRUSTSTORE_PATH);
+ if (keystorePath != null && truststorePath != null) {
+ launchOp.appendArgs("--https",
+ keystorePath.toUri().getPath(),
+ truststorePath.toUri().getPath());
+ } else {
+ launchOp.appendArgs("--http");
+ }
+ launchOp.appendArgs(ctx.getExecutionAttribute(PID_FILE_PATH).toString(),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
ctx.getExecutionAttribute(LOCAL_DIRS)),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index c86ed537874..21660c15634 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -1205,19 +1205,29 @@ private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx,
PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER);
launchOp.appendArgs(runAsUser, ctx.getExecutionAttribute(USER),
- Integer.toString(PrivilegedOperation
- .RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()),
- ctx.getExecutionAttribute(APPID),
- containerIdStr,
- containerWorkDir.toString(),
- nmPrivateContainerScriptPath.toUri().getPath(),
- ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(),
- ctx.getExecutionAttribute(PID_FILE_PATH).toString(),
- StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
- localDirs),
- StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
- logDirs),
- commandFile);
+ Integer.toString(PrivilegedOperation
+ .RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()),
+ ctx.getExecutionAttribute(APPID),
+ containerIdStr,
+ containerWorkDir.toString(),
+ nmPrivateContainerScriptPath.toUri().getPath(),
+ ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath());
+ Path keystorePath = ctx.getExecutionAttribute(NM_PRIVATE_KEYSTORE_PATH);
+ Path truststorePath = ctx.getExecutionAttribute(NM_PRIVATE_TRUSTSTORE_PATH);
+ if (keystorePath != null && truststorePath != null) {
+ launchOp.appendArgs("--https",
+ keystorePath.toUri().getPath(),
+ truststorePath.toUri().getPath());
+ } else {
+ launchOp.appendArgs("--http");
+ }
+ launchOp.appendArgs(
+ ctx.getExecutionAttribute(PID_FILE_PATH).toString(),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ localDirs),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ logDirs),
+ commandFile);
String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
index 2f4aad49a04..fc86b17ed9e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
@@ -58,6 +58,10 @@ private LinuxContainerRuntimeConstants() {
Attribute.attribute(Path.class, "nm_private_container_script_path");
public static final Attribute NM_PRIVATE_TOKENS_PATH = Attribute
.attribute(Path.class, "nm_private_tokens_path");
+ public static final Attribute NM_PRIVATE_KEYSTORE_PATH = Attribute
+ .attribute(Path.class, "nm_private_keystore_path");
+ public static final Attribute NM_PRIVATE_TRUSTSTORE_PATH = Attribute
+ .attribute(Path.class, "nm_private_truststore_path");
public static final Attribute PID_FILE_PATH = Attribute.attribute(
Path.class, "pid_file_path");
public static final Attribute LOCAL_DIRS = Attribute.attribute(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
index ff415727c08..444a1e0a64c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
@@ -40,6 +40,8 @@
private final Map> localizedResources;
private final Path nmPrivateContainerScriptPath;
private final Path nmPrivateTokensPath;
+ private final Path nmPrivateKeystorePath;
+ private final Path nmPrivateTruststorePath;
private final String user;
private final String appId;
private final Path containerWorkDir;
@@ -57,6 +59,8 @@
private Map> localizedResources;
private Path nmPrivateContainerScriptPath;
private Path nmPrivateTokensPath;
+ private Path nmPrivateKeystorePath;
+ private Path nmPrivateTruststorePath;
private String user;
private String appId;
private Path containerWorkDir;
@@ -94,6 +98,16 @@ public Builder setNmPrivateTokensPath(Path nmPrivateTokensPath) {
return this;
}
+ public Builder setNmPrivateKeystorePath(Path nmPrivateKeystorePath) {
+ this.nmPrivateKeystorePath = nmPrivateKeystorePath;
+ return this;
+ }
+
+ public Builder setNmPrivateTruststorePath(Path nmPrivateTruststorePath) {
+ this.nmPrivateTruststorePath = nmPrivateTruststorePath;
+ return this;
+ }
+
public Builder setUser(String user) {
this.user = user;
return this;
@@ -161,6 +175,8 @@ private ContainerStartContext(Builder builder) {
this.localizedResources = builder.localizedResources;
this.nmPrivateContainerScriptPath = builder.nmPrivateContainerScriptPath;
this.nmPrivateTokensPath = builder.nmPrivateTokensPath;
+ this.nmPrivateKeystorePath = builder.nmPrivateKeystorePath;
+ this.nmPrivateTruststorePath = builder.nmPrivateTruststorePath;
this.user = builder.user;
this.appId = builder.appId;
this.containerWorkDir = builder.containerWorkDir;
@@ -194,6 +210,14 @@ public Path getNmPrivateTokensPath() {
return this.nmPrivateTokensPath;
}
+ public Path getNmPrivateKeystorePath() {
+ return this.nmPrivateKeystorePath;
+ }
+
+ public Path getNmPrivateTruststorePath() {
+ return this.nmPrivateTruststorePath;
+ }
+
public String getUser() {
return this.user;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 1f7ae3f944e..6704177070e 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -613,6 +613,16 @@ char *get_container_credentials_file(const char* work_dir) {
CREDENTIALS_FILENAME);
}
+char *get_container_keystore_file(const char* work_dir) {
+ return concatenate("%s/%s", "am container keystore", 2, work_dir,
+ KEYSTORE_FILENAME);
+}
+
+char *get_container_truststore_file(const char* work_dir) {
+ return concatenate("%s/%s", "am container truststore", 2, work_dir,
+ TRUSTSTORE_FILENAME);
+}
+
/**
* Get the app log directory under the given log_root
*/
@@ -1565,9 +1575,11 @@ int exec_docker_command(char *docker_command, char **argv, int argc) {
}
int create_script_paths(const char *work_dir,
- const char *script_name, const char *cred_file,
- char** script_file_dest, char** cred_file_dest,
- int* container_file_source, int* cred_file_source ) {
+ const char *script_name, const char *cred_file, const int https,
+ const char *keystore_file, const char *truststore_file,
+ char** script_file_dest, char** cred_file_dest, char** keystore_file_dest,
+ char** truststore_file_dest, int* container_file_source,
+ int* cred_file_source, int* keystore_file_source, int* truststore_file_source) {
int exit_code = -1;
*script_file_dest = get_container_launcher_file(work_dir);
@@ -1585,6 +1597,24 @@ int create_script_paths(const char *work_dir,
fflush(ERRORFILE);
return exit_code;
}
+
+ if (https == 1) {
+ *keystore_file_dest = get_container_keystore_file(work_dir);
+ if (NULL == keystore_file_dest) {
+ exit_code = OUT_OF_MEMORY;
+ fprintf(ERRORFILE, "Could not create keystore_file_dest");
+ fflush(ERRORFILE);
+ return exit_code;
+ }
+ *truststore_file_dest = get_container_truststore_file(work_dir);
+ if (NULL == truststore_file_dest) {
+ exit_code = OUT_OF_MEMORY;
+ fprintf(ERRORFILE, "Could not create truststore_file_dest");
+ fflush(ERRORFILE);
+ return exit_code;
+ }
+ }
+
// open launch script
*container_file_source = open_file_as_nm(script_name);
if (*container_file_source == -1) {
@@ -1596,12 +1626,31 @@ int create_script_paths(const char *work_dir,
// open credentials
*cred_file_source = open_file_as_nm(cred_file);
if (*cred_file_source == -1) {
- exit_code = INVALID_ARGUMENT_NUMBER;
+ exit_code = INVALID_NM_ROOT_DIRS;
fprintf(ERRORFILE, "Could not open cred file");
fflush(ERRORFILE);
return exit_code;
}
+ if (https == 1) {
+ // open keystore
+ *keystore_file_source = open_file_as_nm(keystore_file);
+ if (*keystore_file_source == -1) {
+ exit_code = INVALID_NM_ROOT_DIRS;
+ fprintf(ERRORFILE, "Could not open keystore file");
+ fflush(ERRORFILE);
+ return exit_code;
+ }
+ // open truststore
+ *truststore_file_source = open_file_as_nm(truststore_file);
+ if (*truststore_file_source == -1) {
+ exit_code = INVALID_NM_ROOT_DIRS;
+ fprintf(ERRORFILE, "Could not open truststore file");
+ fflush(ERRORFILE);
+ return exit_code;
+ }
+ }
+
exit_code = 0;
return exit_code;
}
@@ -1609,10 +1658,14 @@ int create_script_paths(const char *work_dir,
int create_local_dirs(const char * user, const char *app_id,
const char *container_id, const char *work_dir,
const char *script_name, const char *cred_file,
+ const int https,
+ const char *keystore_file, const char *truststore_file,
char* const* local_dirs,
char* const* log_dirs, int effective_user,
char* script_file_dest, char* cred_file_dest,
- int container_file_source, int cred_file_source) {
+ char* keystore_file_dest, char* truststore_file_dest,
+ int container_file_source, int cred_file_source,
+ int keystore_file_source, int truststore_file_source) {
int exit_code = -1;
// create the user directory on all disks
int result = initialize_user(user, local_dirs);
@@ -1665,12 +1718,32 @@ int create_local_dirs(const char * user, const char *app_id,
// Copy credential file to permissions 600
if (copy_file(cred_file_source, cred_file, cred_file_dest,
S_IRUSR | S_IWUSR) != 0) {
- exit_code = COULD_NOT_CREATE_CREDENTIALS_FILE;
+ exit_code = COULD_NOT_CREATE_CREDENTIALS_COPY;
fprintf(ERRORFILE, "Could not copy file");
fflush(ERRORFILE);
goto cleanup;
}
+ if (https == 1) {
+ // Copy keystore file to permissions 600
+ if (copy_file(keystore_file_source, keystore_file, keystore_file_dest,
+ S_IRUSR | S_IWUSR) != 0) {
+ exit_code = COULD_NOT_CREATE_KEYSTORE_COPY;
+ fprintf(ERRORFILE, "Could not copy file");
+ fflush(ERRORFILE);
+ goto cleanup;
+ }
+
+ // Copy truststore file to permissions 600
+ if (copy_file(truststore_file_source, truststore_file, truststore_file_dest,
+ S_IRUSR | S_IWUSR) != 0) {
+ exit_code = COULD_NOT_CREATE_TRUSTSTORE_COPY;
+ fprintf(ERRORFILE, "Could not copy file");
+ fflush(ERRORFILE);
+ goto cleanup;
+ }
+ }
+
if (chdir(work_dir) != 0) {
fprintf(ERRORFILE, "Can't change directory to %s -%s\n", work_dir,
strerror(errno));
@@ -1708,17 +1781,23 @@ int create_user_filecache_dirs(const char * user, char* const* local_dirs) {
int launch_docker_container_as_user(const char * user, const char *app_id,
const char *container_id, const char *work_dir,
const char *script_name, const char *cred_file,
+ const int https,
+ const char *keystore_file, const char *truststore_file,
const char *pid_file, char* const* local_dirs,
char* const* log_dirs, const char *command_file) {
int exit_code = -1;
char *script_file_dest = NULL;
char *cred_file_dest = NULL;
+ char *keystore_file_dest = NULL;
+ char *truststore_file_dest = NULL;
char *exit_code_file = NULL;
char *docker_command_with_binary = NULL;
char *docker_inspect_command = NULL;
char *docker_inspect_exitcode_command = NULL;
int container_file_source =-1;
int cred_file_source = -1;
+ int keystore_file_source = -1;
+ int truststore_file_source = -1;
int use_entry_point = 0;
gid_t user_gid = getegid();
@@ -1729,8 +1808,8 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
fprintf(LOGFILE, "Creating script paths...\n");
exit_code = create_script_paths(
- work_dir, script_name, cred_file, &script_file_dest, &cred_file_dest,
- &container_file_source, &cred_file_source);
+ work_dir, script_name, cred_file, https, keystore_file, truststore_file, &script_file_dest, &cred_file_dest,
+ &keystore_file_dest, &truststore_file_dest, &container_file_source, &cred_file_source, &keystore_file_source, &truststore_file_source);
if (exit_code != 0) {
fprintf(ERRORFILE, "Could not create script path\n");
fflush(ERRORFILE);
@@ -1739,9 +1818,9 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
fprintf(LOGFILE, "Creating local dirs...\n");
exit_code = create_local_dirs(user, app_id, container_id,
- work_dir, script_name, cred_file, local_dirs, log_dirs,
- 1, script_file_dest, cred_file_dest,
- container_file_source, cred_file_source);
+ work_dir, script_name, cred_file, https, keystore_file, truststore_file, local_dirs, log_dirs,
+ 1, script_file_dest, cred_file_dest, keystore_file_dest, truststore_file_dest,
+ container_file_source, cred_file_source, keystore_file_source, truststore_file_source);
if (exit_code != 0) {
fprintf(ERRORFILE, "Could not create local files and directories %d %d\n", container_file_source, cred_file_source);
fflush(ERRORFILE);
@@ -1973,6 +2052,8 @@ cleanup:
free(exit_code_file);
free(script_file_dest);
free(cred_file_dest);
+ free(keystore_file_dest);
+ free(truststore_file_dest);
free(docker_command_with_binary);
free(docker_inspect_command);
free_values(docker_command);
@@ -1983,12 +2064,16 @@ cleanup:
int launch_container_as_user(const char *user, const char *app_id,
const char *container_id, const char *work_dir,
const char *script_name, const char *cred_file,
+ const int https,
+ const char *keystore_file, const char *truststore_file,
const char* pid_file, char* const* local_dirs,
char* const* log_dirs, const char *resources_key,
char* const* resources_values) {
int exit_code = -1;
char *script_file_dest = NULL;
char *cred_file_dest = NULL;
+ char *keystore_file_dest = NULL;
+ char *truststore_file_dest = NULL;
char *exit_code_file = NULL;
fprintf(LOGFILE, "Getting exit code file...\n");
@@ -2000,11 +2085,13 @@ int launch_container_as_user(const char *user, const char *app_id,
int container_file_source =-1;
int cred_file_source = -1;
+ int keystore_file_source = -1;
+ int truststore_file_source = -1;
fprintf(LOGFILE, "Creating script paths...\n");
exit_code = create_script_paths(
- work_dir, script_name, cred_file, &script_file_dest, &cred_file_dest,
- &container_file_source, &cred_file_source);
+ work_dir, script_name, cred_file, https, keystore_file, truststore_file, &script_file_dest, &cred_file_dest,
+ &keystore_file_dest, &truststore_file_dest, &container_file_source, &cred_file_source, &keystore_file_source, &truststore_file_source);
if (exit_code != 0) {
fprintf(ERRORFILE, "Could not create local files and directories");
fflush(ERRORFILE);
@@ -2052,9 +2139,9 @@ int launch_container_as_user(const char *user, const char *app_id,
fprintf(LOGFILE, "Creating local dirs...\n");
exit_code = create_local_dirs(user, app_id, container_id,
- work_dir, script_name, cred_file, local_dirs, log_dirs,
- 0, script_file_dest, cred_file_dest,
- container_file_source, cred_file_source);
+ work_dir, script_name, cred_file, https, keystore_file, truststore_file, local_dirs, log_dirs,
+ 0, script_file_dest, cred_file_dest, keystore_file_dest, truststore_file_dest,
+ container_file_source, cred_file_source, keystore_file_source, truststore_file_source);
if (exit_code != 0) {
fprintf(ERRORFILE, "Could not create local files and directories");
fflush(ERRORFILE);
@@ -2087,6 +2174,8 @@ int launch_container_as_user(const char *user, const char *app_id,
free(exit_code_file);
free(script_file_dest);
free(cred_file_dest);
+ free(keystore_file_dest);
+ free(truststore_file_dest);
return exit_code;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index ce05c573c95..3eb931ae099 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -59,6 +59,8 @@ enum operations {
#define CONTAINER_DIR_PATTERN NM_APP_DIR_PATTERN "/%s"
#define CONTAINER_SCRIPT "launch_container.sh"
#define CREDENTIALS_FILENAME "container_tokens"
+#define KEYSTORE_FILENAME "yarn_provided.keystore"
+#define TRUSTSTORE_FILENAME "yarn_provided.truststore"
#define MIN_USERID_KEY "min.user.id"
#define BANNED_USERS_KEY "banned.users"
#define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users"
@@ -102,6 +104,8 @@ int initialize_app(const char *user, const char *app_id,
int launch_docker_container_as_user(const char * user, const char *app_id,
const char *container_id, const char *work_dir,
const char *script_name, const char *cred_file,
+ const int https,
+ const char *keystore_file, const char *truststore_file,
const char *pid_file, char* const* local_dirs,
char* const* log_dirs,
const char *command_file);
@@ -118,8 +122,13 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
* @param container_id the container id
* @param work_dir the working directory for the container.
* @param script_name the name of the script to be run to launch the container.
- * @param cred_file the credentials file that needs to be compied to the
+ * @param cred_file the credentials file that needs to be copied to the
* working directory.
+ * @param https 1 if a keystore and truststore will be provided, 0 if not
+ * @param keystore_file the keystore file that needs to be copied to the
+ * working directory.
+ * @param truststore_file the truststore file that needs to be copied to the
+ * working directory
* @param pid_file file where pid of process should be written to
* @param local_dirs nodemanager-local-directories to be used
* @param log_dirs nodemanager-log-directories to be used
@@ -130,6 +139,8 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
int launch_container_as_user(const char * user, const char *app_id,
const char *container_id, const char *work_dir,
const char *script_name, const char *cred_file,
+ const int https,
+ const char *keystore_file, const char *truststore_file,
const char *pid_file, char* const* local_dirs,
char* const* log_dirs, const char *resources_key,
char* const* resources_value);
@@ -194,6 +205,10 @@ char *get_container_launcher_file(const char* work_dir);
char *get_container_credentials_file(const char* work_dir);
+char *get_container_keystore_file(const char* work_dir);
+
+char *get_container_truststore_file(const char* work_dir);
+
/**
* Get the app log directory under log_root
*/
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index a3057e63998..c269fa4246b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -227,6 +227,9 @@ static struct {
char **resources_values;
const char *app_id;
const char *container_id;
+ int https;
+ const char *keystore_file;
+ const char *truststore_file;
const char *cred_file;
const char *script_file;
const char *current_dir;
@@ -432,8 +435,8 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
case LAUNCH_DOCKER_CONTAINER:
if(is_docker_support_enabled()) {
//kill me now.
- if (!(argc == 13 || argc == 14)) {
- fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14) for"
+ if (!(argc >= 14 && argc <= 17)) {
+ fprintf(ERRORFILE, "Wrong number of arguments (%d vs 14 - 17) for"
" launch docker container\n", argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
@@ -444,6 +447,13 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.current_dir = argv[optind++];
cmd_input.script_file = argv[optind++];
cmd_input.cred_file = argv[optind++];
+ if (strcmp("--https", argv[optind++]) == 0) {
+ cmd_input.https = 1;
+ cmd_input.keystore_file = argv[optind++];
+ cmd_input.truststore_file = argv[optind++];
+ } else {
+ cmd_input.https = 0;
+ }
cmd_input.pid_file = argv[optind++];
// good local dirs as a comma separated list
cmd_input.local_dirs = argv[optind++];
@@ -451,7 +461,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.log_dirs = argv[optind++];
cmd_input.docker_command_file = argv[optind++];
//network isolation through tc
- if (argc == 14) {
+ if ((argc == 15 && !cmd_input.https) || (argc == 17 && cmd_input.https)) {
if(is_tc_support_enabled()) {
cmd_input.traffic_control_command_file = argv[optind++];
} else {
@@ -469,8 +479,8 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
case LAUNCH_CONTAINER:
//kill me now.
- if (!(argc == 13 || argc == 14)) {
- fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14)"
+ if (!(argc >= 14 && argc <= 17)) {
+ fprintf(ERRORFILE, "Wrong number of arguments (%d vs 14 - 17)"
" for launch container\n", argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
@@ -481,6 +491,13 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.current_dir = argv[optind++];
cmd_input.script_file = argv[optind++];
cmd_input.cred_file = argv[optind++];
+ if (strcmp("--https", argv[optind++]) == 0) {
+ cmd_input.https = 1;
+ cmd_input.keystore_file = argv[optind++];
+ cmd_input.truststore_file = argv[optind++];
+ } else {
+ cmd_input.https = 0;
+ }
cmd_input.pid_file = argv[optind++];
cmd_input.local_dirs = argv[optind++];// good local dirs as a comma separated list
cmd_input.log_dirs = argv[optind++];// good log dirs as a comma separated list
@@ -499,7 +516,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
}
//network isolation through tc
- if (argc == 14) {
+ if ((argc == 15 && !cmd_input.https) || (argc == 17 && cmd_input.https)) {
if(is_tc_support_enabled()) {
cmd_input.traffic_control_command_file = argv[optind++];
} else {
@@ -636,6 +653,9 @@ int main(int argc, char **argv) {
cmd_input.current_dir,
cmd_input.script_file,
cmd_input.cred_file,
+ cmd_input.https,
+ cmd_input.keystore_file,
+ cmd_input.truststore_file,
cmd_input.pid_file,
split(cmd_input.local_dirs),
split(cmd_input.log_dirs),
@@ -662,6 +682,9 @@ int main(int argc, char **argv) {
cmd_input.current_dir,
cmd_input.script_file,
cmd_input.cred_file,
+ cmd_input.https,
+ cmd_input.keystore_file,
+ cmd_input.truststore_file,
cmd_input.pid_file,
split(cmd_input.local_dirs),
split(cmd_input.log_dirs),
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
index 372c17a4f80..5702b0c9d50 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -59,7 +59,7 @@ enum errorcodes {
ERROR_READING_DOCKER_FILE = 31,
FEATURE_DISABLED = 32,
COULD_NOT_CREATE_SCRIPT_COPY = 33,
- COULD_NOT_CREATE_CREDENTIALS_FILE = 34,
+ COULD_NOT_CREATE_CREDENTIALS_COPY = 34,
COULD_NOT_CREATE_WORK_DIRECTORIES = 35,
COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36,
COULD_NOT_CREATE_TMP_DIRECTORIES = 37,
@@ -69,7 +69,9 @@ enum errorcodes {
// DOCKER_CONTAINER_NAME_INVALID = 41, (NOT USED)
ERROR_COMPILING_REGEX = 42,
INVALID_CONTAINER_ID = 43,
- DOCKER_EXEC_FAILED = 44
+ DOCKER_EXEC_FAILED = 44,
+ COULD_NOT_CREATE_KEYSTORE_COPY = 45,
+ COULD_NOT_CREATE_TRUSTSTORE_COPY = 46
};
/* Macros for min/max. */
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index f4f00c0e059..a4515ce29c8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -168,13 +168,13 @@ void test_get_user_directory() {
void test_check_nm_local_dir() {
// check filesystem is same as running user.
int expected = 0;
- char *local_path = TEST_ROOT "target";
+ char *local_path = TEST_ROOT "/target";
char *root_path = "/";
if (mkdirs(local_path, 0700) != 0) {
printf("FAIL: unble to create node manager local directory: %s\n", local_path);
exit(1);
}
- int actual = check_nm_local_dir(nm_uid, local_path);
+ int actual = check_nm_local_dir(user_detail->pw_uid, local_path);
if (expected != actual) {
printf("test_nm_local_dir expected %d got %d\n", expected, actual);
exit(1);
@@ -199,31 +199,76 @@ void test_get_app_directory() {
free(app_dir);
}
-void test_get_container_directory() {
- char *container_dir = get_container_work_directory(TEST_ROOT, "owen", "app_1",
+void test_get_container_work_directory() {
+ char *expected_file = TEST_ROOT "/usercache/user/appcache/app_1/container_1";
+ char *work_dir = get_container_work_directory(TEST_ROOT, "user", "app_1",
"container_1");
- char *expected = TEST_ROOT "/usercache/owen/appcache/app_1/container_1";
- if (strcmp(container_dir, expected) != 0) {
- printf("Fail get_container_work_directory got %s expected %s\n",
- container_dir, expected);
+ if (strcmp(work_dir, expected_file) != 0) {
+ printf("Fail get_container_work_directory expected %s got %s\n",
+ expected_file, work_dir);
exit(1);
}
- free(container_dir);
+ free(work_dir);
}
void test_get_container_launcher_file() {
- char *expected_file = (TEST_ROOT "/usercache/user/appcache/app_200906101234_0001"
- "/launch_container.sh");
- char *app_dir = get_app_directory(TEST_ROOT, "user",
- "app_200906101234_0001");
- char *container_file = get_container_launcher_file(app_dir);
- if (strcmp(container_file, expected_file) != 0) {
- printf("failure to match expected container file %s vs %s\n", container_file,
- expected_file);
+ char *expected_file = (TEST_ROOT "/usercache/user/appcache/"
+ "app_200906101234_0001/container_1/launch_container.sh");
+ char *work_dir = get_container_work_directory(TEST_ROOT, "user",
+ "app_200906101234_0001", "container_1");
+ char *launcher_file = get_container_launcher_file(work_dir);
+ if (strcmp(launcher_file, expected_file) != 0) {
+ printf("failure to match expected launcher file %s got %s\n",
+ expected_file, launcher_file);
+ exit(1);
+ }
+ free(work_dir);
+ free(launcher_file);
+}
+
+void test_get_container_credentials_file() {
+ char *expected_file = (TEST_ROOT "/usercache/user/appcache/"
+ "app_200906101234_0001/container_1/container_tokens");
+ char *work_dir = get_container_work_directory(TEST_ROOT, "user",
+ "app_200906101234_0001", "container_1");
+ char *credentials_file = get_container_credentials_file(work_dir);
+ if (strcmp(credentials_file, expected_file) != 0) {
+ printf("failure to match expected credentials file %s got %s\n",
+ expected_file, credentials_file);
exit(1);
}
- free(app_dir);
- free(container_file);
+ free(work_dir);
+ free(credentials_file);
+}
+
+void test_get_container_keystore_file() {
+ char *expected_file = (TEST_ROOT "/usercache/user/appcache/"
+ "app_200906101234_0001/container_1/yarn_provided.keystore");
+ char *work_dir = get_container_work_directory(TEST_ROOT, "user",
+ "app_200906101234_0001", "container_1");
+ char *keystore_file = get_container_keystore_file(work_dir);
+ if (strcmp(keystore_file, expected_file) != 0) {
+ printf("failure to match expected keystore file %s got %s\n",
+ expected_file, keystore_file);
+ exit(1);
+ }
+ free(work_dir);
+ free(keystore_file);
+}
+
+void test_get_container_truststore_file() {
+ char *expected_file = (TEST_ROOT "/usercache/user/appcache/"
+ "app_200906101234_0001/container_1/yarn_provided.truststore");
+ char *work_dir = get_container_work_directory(TEST_ROOT, "user",
+ "app_200906101234_0001", "container_1");
+ char *truststore_file = get_container_truststore_file(work_dir);
+ if (strcmp(truststore_file, expected_file) != 0) {
+ printf("failure to match expected truststore file %s got %s\n",
+ expected_file, truststore_file);
+ exit(1);
+ }
+ free(work_dir);
+ free(truststore_file);
}
void test_get_app_log_dir() {
@@ -762,38 +807,31 @@ void test_signal_container_group() {
}
}
-void test_init_app() {
- printf("\nTesting init app\n");
- if (seteuid(0) != 0) {
- printf("FAIL: seteuid to root failed - %s\n", strerror(errno));
- exit(1);
- }
- FILE* creds = fopen(TEST_ROOT "/creds.txt", "w");
+void create_text_file(const char* filename, const char* contents) {
+ FILE* creds = fopen(filename, "w");
if (creds == NULL) {
- printf("FAIL: failed to create credentials file - %s\n", strerror(errno));
+ printf("FAIL: failed to create %s file - %s\n", filename, strerror(errno));
exit(1);
}
- if (fprintf(creds, "secret key\n") < 0) {
- printf("FAIL: fprintf failed - %s\n", strerror(errno));
+ if (fwrite(contents, sizeof(char), sizeof(contents), creds)
+ < sizeof(contents)) {
+ printf("FAIL: fwrite failed on file %s- %s\n", filename, strerror(errno));
exit(1);
}
if (fclose(creds) != 0) {
- printf("FAIL: fclose failed - %s\n", strerror(errno));
- exit(1);
- }
- FILE* job_xml = fopen(TEST_ROOT "/job.xml", "w");
- if (job_xml == NULL) {
- printf("FAIL: failed to create job file - %s\n", strerror(errno));
- exit(1);
- }
- if (fprintf(job_xml, "\n") < 0) {
- printf("FAIL: fprintf failed - %s\n", strerror(errno));
+ printf("FAIL: fclose failed on file %s - %s\n", filename, strerror(errno));
exit(1);
}
- if (fclose(job_xml) != 0) {
- printf("FAIL: fclose failed - %s\n", strerror(errno));
+}
+
+void test_init_app() {
+ printf("\nTesting init app\n");
+ if (seteuid(0) != 0) {
+ printf("FAIL: seteuid to root failed - %s\n", strerror(errno));
exit(1);
}
+ create_text_file(TEST_ROOT "/creds.txt", "secret key");
+ create_text_file(TEST_ROOT "/job.xml", "\n");
if (seteuid(user_detail->pw_uid) != 0) {
printf("FAIL: failed to seteuid back to user - %s\n", strerror(errno));
exit(1);
@@ -807,14 +845,9 @@ void test_init_app() {
exit(1);
} else if (child == 0) {
char *final_pgm[] = {"touch", "my-touch-file", 0};
- if (initialize_app(yarn_username, "app_4", "container_1",
+ exit(initialize_app(yarn_username, "app_4", "container_1",
TEST_ROOT "/creds.txt",
- local_dirs, log_dirs, final_pgm) != 0) {
- printf("FAIL: failed in child\n");
- exit(42);
- }
- // should never return
- exit(1);
+ local_dirs, log_dirs, final_pgm));
}
int status = 0;
if (waitpid(child, &status, 0) <= 0) {
@@ -822,6 +855,11 @@ void test_init_app() {
strerror(errno));
exit(1);
}
+ if (WEXITSTATUS(status) != 0) {
+ printf("FAIL: child %" PRId64 " exited with bad status %d\n",
+ (int64_t)child, WEXITSTATUS(status));
+ exit(1);
+ }
if (access(TEST_ROOT "/logs/userlogs/app_4", R_OK) != 0) {
printf("FAIL: failed to create app log directory\n");
exit(1);
@@ -859,24 +897,24 @@ void test_init_app() {
free(container_dir);
}
-void test_run_container() {
- printf("\nTesting run container\n");
+void test_launch_container(const char* app, int https) {
+ if (https == 1) {
+ printf("\nTesting launch container with HTTPS\n");
+ } else {
+ printf("\nTesting launch container without HTTPS\n");
+ }
if (seteuid(0) != 0) {
printf("FAIL: seteuid to root failed - %s\n", strerror(errno));
exit(1);
}
- FILE* creds = fopen(TEST_ROOT "/creds.txt", "w");
- if (creds == NULL) {
- printf("FAIL: failed to create credentials file - %s\n", strerror(errno));
- exit(1);
- }
- if (fprintf(creds, "secret key\n") < 0) {
- printf("FAIL: fprintf failed - %s\n", strerror(errno));
- exit(1);
- }
- if (fclose(creds) != 0) {
- printf("FAIL: fclose failed - %s\n", strerror(errno));
- exit(1);
+ create_text_file(TEST_ROOT "/creds.txt", "secret key");
+ char* keystore_file = NULL;
+ char* truststore_file = NULL;
+ if (https == 1) {
+ keystore_file = TEST_ROOT "/yarn_provided.keystore";
+ truststore_file = TEST_ROOT "/yarn_provided.truststore";
+ create_text_file(keystore_file, "keystore");
+ create_text_file(truststore_file, "truststore");
}
char * cgroups_pids[] = { TEST_ROOT "/cgroups-pid1.txt", TEST_ROOT "/cgroups-pid2.txt", 0 };
@@ -906,32 +944,36 @@ void test_run_container() {
fflush(stdout);
fflush(stderr);
char* container_dir = get_container_work_directory(TEST_ROOT "/local-1",
- yarn_username, "app_4", "container_1");
+ yarn_username, app, "container_1");
const char * pid_file = TEST_ROOT "/pid.txt";
pid_t child = fork();
if (child == -1) {
printf("FAIL: failed to fork process for init_app - %s\n",
- strerror(errno));
+ strerror(errno));
exit(1);
} else if (child == 0) {
- if (launch_container_as_user(yarn_username, "app_4", "container_1",
- container_dir, script_name, TEST_ROOT "/creds.txt", pid_file,
- local_dirs, log_dirs,
- "cgroups", cgroups_pids) != 0) {
- printf("FAIL: failed in child\n");
- exit(42);
- }
- // should never return
- exit(1);
+ exit(launch_container_as_user(yarn_username, app, "container_1",
+ container_dir, script_name, TEST_ROOT "/creds.txt",
+ https, keystore_file, truststore_file,
+ pid_file, local_dirs, log_dirs,
+ "cgroups", cgroups_pids));
}
int status = 0;
if (waitpid(child, &status, 0) <= 0) {
printf("FAIL: failed waiting for process %" PRId64 " - %s\n", (int64_t)child,
- strerror(errno));
+ strerror(errno));
+ exit(1);
+ }
+ if (WEXITSTATUS(status) != 0) {
+ printf("FAIL: child %" PRId64 " exited with bad status %d\n",
+ (int64_t)child, WEXITSTATUS(status));
exit(1);
}
- if (access(TEST_ROOT "/logs/userlogs/app_4/container_1", R_OK) != 0) {
+ char container_log_path[100000];
+ snprintf(container_log_path, sizeof container_log_path, "%s%s%s%s", TEST_ROOT,
+ "/logs/userlogs/", app, "/container_1");
+ if (access(container_log_path, R_OK) != 0) {
printf("FAIL: failed to create container log directory\n");
exit(1);
}
@@ -939,14 +981,17 @@ void test_run_container() {
printf("FAIL: failed to create container directory %s\n", container_dir);
exit(1);
}
- char buffer[100000];
- sprintf(buffer, "%s/foobar", container_dir);
- if (access(buffer, R_OK) != 0) {
- printf("FAIL: failed to create touch file %s\n", buffer);
+ char touchfile[100000];
+ sprintf(touchfile, "%s/foobar", container_dir);
+ if (access(touchfile, R_OK) != 0) {
+ printf("FAIL: failed to create touch file %s\n", touchfile);
exit(1);
}
free(container_dir);
- container_dir = get_app_log_directory(TEST_ROOT "/logs/userlogs", "app_4/container_1");
+ char app_log_path[100000];
+ snprintf(app_log_path, sizeof app_log_path, "%s%s%s", TEST_ROOT,
+ "/logs/userlogs/", app);
+ container_dir = get_app_log_directory(app_log_path, "container_1");
if (access(container_dir, R_OK) != 0) {
printf("FAIL: failed to create app log directory %s\n", container_dir);
exit(1);
@@ -1395,6 +1440,13 @@ int main(int argc, char **argv) {
exit(1);
}
+ if (mkdirs(TEST_ROOT, 0777) != 0) {
+ exit(1);
+ }
+ if (chmod(TEST_ROOT, 0777) != 0) { // in case of umask
+ exit(1);
+ }
+
if (mkdirs(TEST_ROOT "/logs/userlogs", 0755) != 0) {
exit(1);
}
@@ -1448,12 +1500,21 @@ int main(int argc, char **argv) {
printf("\nTesting get_app_directory()\n");
test_get_app_directory();
- printf("\nTesting get_container_directory()\n");
- test_get_container_directory();
+ printf("\nTesting get_container_work_directory()\n");
+ test_get_container_work_directory();
printf("\nTesting get_container_launcher_file()\n");
test_get_container_launcher_file();
+ printf("\nTesting get_container_credentials_file()\n");
+ test_get_container_credentials_file();
+
+ printf("\nTesting get_container_keystore_file()\n");
+ test_get_container_keystore_file();
+
+ printf("\nTesting get_container_truststore_file()\n");
+ test_get_container_truststore_file();
+
printf("\nTesting get_app_log_dir()\n");
test_get_app_log_dir();
@@ -1510,7 +1571,8 @@ int main(int argc, char **argv) {
// these tests do internal forks so that the change_owner and execs
// don't mess up our process.
test_init_app();
- test_run_container();
+ test_launch_container("app_4", 0);
+ test_launch_container("app_5", 1);
}
/*
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
index 9b180c7eff6..7ec3ae3b3e4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
@@ -47,6 +47,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
@@ -69,6 +70,7 @@
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.MockLocalizerHeartbeatResponse;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
@@ -225,6 +227,157 @@ public void testDirPermissions() throws Exception {
}
}
+ private void writeStringToRelativePath(FileContext fc, Path p, String str)
+ throws IOException {
+ p = p.makeQualified(fc.getDefaultFileSystem().getUri(),
+ new Path(new File(".").getAbsolutePath()));
+ try (FSDataOutputStream os = fc.create(p).build()) {
+ os.writeUTF(str);
+ }
+ }
+
+ private String readStringFromPath(FileContext fc, Path p) throws IOException {
+ try (FSDataInputStream is = fc.open(p)) {
+ return is.readUTF();
+ }
+ }
+
+ @Test
+ public void testLaunchContainerCopyFilesWithoutHTTPS() throws Exception {
+ testLaunchContainerCopyFiles(false);
+ }
+
+ @Test
+ public void testLaunchContainerCopyFilesWithHTTPS() throws Exception {
+ testLaunchContainerCopyFiles(true);
+ }
+
+ private void testLaunchContainerCopyFiles(boolean https) throws Exception {
+ if (Shell.WINDOWS) {
+ BASE_TMP_PATH =
+ new Path(new File("target").getAbsolutePath(),
+ TestDefaultContainerExecutor.class.getSimpleName());
+ }
+
+ Path localDir = new Path(BASE_TMP_PATH, "localDir");
+ List localDirs = new ArrayList();
+ localDirs.add(localDir.toString());
+ List logDirs = new ArrayList();
+ Path logDir = new Path(BASE_TMP_PATH, "logDir");
+ logDirs.add(logDir.toString());
+
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.toString());
+ conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.toString());
+
+ FileContext lfs = FileContext.getLocalFSFileContext(conf);
+ deleteTmpFiles();
+ lfs.mkdir(BASE_TMP_PATH, FsPermission.getDefault(), true);
+ DefaultContainerExecutor dce = new DefaultContainerExecutor(lfs);
+ dce.setConf(conf);
+
+ Container container = mock(Container.class);
+ ContainerId cId = mock(ContainerId.class);
+ ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+ HashMap env = new HashMap();
+ env.put("LANG", "C");
+
+ String appSubmitter = "nobody";
+ String appId = "APP_ID";
+ String containerId = "CONTAINER_ID";
+
+ when(container.getContainerId()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(context);
+ when(cId.toString()).thenReturn(containerId);
+ when(cId.getApplicationAttemptId()).thenReturn(
+ ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 0));
+ when(context.getEnvironment()).thenReturn(env);
+
+ Path scriptPath = new Path(BASE_TMP_PATH, "script");
+ Path tokensPath = new Path(BASE_TMP_PATH, "tokens");
+ Path keystorePath = new Path(BASE_TMP_PATH, "keystore");
+ Path truststorePath = new Path(BASE_TMP_PATH, "truststore");
+ writeStringToRelativePath(lfs, scriptPath, "script");
+ writeStringToRelativePath(lfs, tokensPath, "tokens");
+ if (https) {
+ writeStringToRelativePath(lfs, keystorePath, "keystore");
+ writeStringToRelativePath(lfs, truststorePath, "truststore");
+ }
+
+ Path workDir = localDir;
+ Path pidFile = new Path(workDir, "pid.txt");
+
+ dce.init(null);
+ dce.activateContainer(cId, pidFile);
+ ContainerStartContext.Builder ctxBuilder =
+ new ContainerStartContext.Builder()
+ .setContainer(container)
+ .setNmPrivateContainerScriptPath(scriptPath)
+ .setNmPrivateTokensPath(tokensPath)
+ .setUser(appSubmitter)
+ .setAppId(appId)
+ .setContainerWorkDir(workDir)
+ .setLocalDirs(localDirs)
+ .setLogDirs(logDirs);
+ if (https) {
+ ctxBuilder.setNmPrivateTruststorePath(truststorePath)
+ .setNmPrivateKeystorePath(keystorePath);
+ }
+ ContainerStartContext ctx = ctxBuilder.build();
+
+ // #launchContainer will copy a number of files to this directory.
+ // Ensure that it doesn't exist first
+ lfs.delete(workDir, true);
+ try {
+ lfs.getFileStatus(workDir);
+ Assert.fail("Expected FileNotFoundException on " + workDir);
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+
+ dce.launchContainer(ctx);
+
+ Path finalScriptPath = new Path(workDir,
+ ContainerLaunch.CONTAINER_SCRIPT);
+ Path finalTokensPath = new Path(workDir,
+ ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE);
+ Path finalKeystorePath = new Path(workDir,
+ ContainerLaunch.KEYSTORE_FILE);
+ Path finalTrustorePath = new Path(workDir,
+ ContainerLaunch.TRUSTSTORE_FILE);
+
+ Assert.assertTrue(lfs.getFileStatus(workDir).isDirectory());
+ Assert.assertTrue(lfs.getFileStatus(finalScriptPath).isFile());
+ Assert.assertTrue(lfs.getFileStatus(finalTokensPath).isFile());
+ if (https) {
+ Assert.assertTrue(lfs.getFileStatus(finalKeystorePath).isFile());
+ Assert.assertTrue(lfs.getFileStatus(finalTrustorePath).isFile());
+ } else {
+ try {
+ lfs.getFileStatus(finalKeystorePath);
+ Assert.fail("Expected FileNotFoundException on " + finalKeystorePath);
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ try {
+ lfs.getFileStatus(finalTrustorePath);
+ Assert.fail("Expected FileNotFoundException on " + finalKeystorePath);
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+
+ Assert.assertEquals("script", readStringFromPath(lfs, finalScriptPath));
+ Assert.assertEquals("tokens", readStringFromPath(lfs, finalTokensPath));
+ if (https) {
+ Assert.assertEquals("keystore", readStringFromPath(lfs,
+ finalKeystorePath));
+ Assert.assertEquals("truststore", readStringFromPath(lfs,
+ finalTrustorePath));
+ }
+ }
+
@Test
public void testContainerLaunchError()
throws IOException, InterruptedException, ConfigurationException {
@@ -303,6 +456,8 @@ public Object answer(InvocationOnMock invocationOnMock)
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
+ Path keystorePath = new Path("file:///dev/null");
+ Path truststorePath = new Path("file:///dev/null");
if (Shell.WINDOWS) {
File tmp = new File(BASE_TMP_PATH.toString(), "test_echo.cmd");
BufferedWriter output = new BufferedWriter(new FileWriter(tmp));
@@ -323,6 +478,8 @@ public Object answer(InvocationOnMock invocationOnMock)
.setContainer(container)
.setNmPrivateContainerScriptPath(scriptPath)
.setNmPrivateTokensPath(tokensPath)
+ .setNmPrivateKeystorePath(keystorePath)
+ .setNmPrivateTruststorePath(truststorePath)
.setUser(appSubmitter)
.setAppId(appId)
.setContainerWorkDir(workDir)
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index a110f103314..2304501cb69 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -37,6 +37,8 @@
import java.io.IOException;
import java.io.LineNumberReader;
import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -88,12 +90,11 @@
private static final Logger LOG =
LoggerFactory.getLogger(TestLinuxContainerExecutorWithMocks.class);
- private static final String MOCK_EXECUTOR =
- "./src/test/resources/mock-container-executor";
+ private static final String MOCK_EXECUTOR = "mock-container-executor";
private static final String MOCK_EXECUTOR_WITH_ERROR =
- "./src/test/resources/mock-container-executer-with-error";
+ "mock-container-executer-with-error";
private static final String MOCK_EXECUTOR_WITH_CONFIG_ERROR =
- "./src/test/resources/mock-container-executer-with-configuration-error";
+ "mock-container-executer-with-configuration-error";
private String tmpMockExecutor;
private LinuxContainerExecutor mockExec = null;
@@ -121,11 +122,13 @@ private void deleteMockParamFile() {
return ret;
}
- private void setupMockExecutor(String executorPath, Configuration conf)
- throws IOException {
+ private void setupMockExecutor(String executorName, Configuration conf)
+ throws IOException, URISyntaxException {
//we'll always use the tmpMockExecutor - since
// PrivilegedOperationExecutor can only be initialized once.
+ URI executorPath = getClass().getClassLoader().getResource(executorName)
+ .toURI();
Files.copy(Paths.get(executorPath), Paths.get(tmpMockExecutor),
REPLACE_EXISTING);
@@ -140,7 +143,8 @@ private void setupMockExecutor(String executorPath, Configuration conf)
}
@Before
- public void setup() throws IOException, ContainerExecutionException {
+ public void setup() throws IOException, ContainerExecutionException,
+ URISyntaxException {
assumeNotWindows();
tmpMockExecutor = System.getProperty("test.build.data") +
@@ -172,7 +176,18 @@ public void tearDown() {
}
@Test
- public void testContainerLaunch()
+ public void testContainerLaunchWithoutHTTPS()
+ throws IOException, ConfigurationException {
+ testContainerLaunch(false);
+ }
+
+ @Test
+ public void testContainerLaunchWithHTTPS()
+ throws IOException, ConfigurationException {
+ testContainerLaunch(true);
+ }
+
+ private void testContainerLaunch(boolean https)
throws IOException, ConfigurationException {
String appSubmitter = "nobody";
String cmd = String.valueOf(
@@ -193,41 +208,64 @@ public void testContainerLaunch()
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
+ Path keystorePath = new Path("file:///dev/null");
+ Path truststorePath = new Path("file:///dev/null");
Path workDir = new Path("/tmp");
Path pidFile = new Path(workDir, "pid.txt");
mockExec.activateContainer(cId, pidFile);
- int ret = mockExec.launchContainer(new ContainerStartContext.Builder()
- .setContainer(container)
- .setNmPrivateContainerScriptPath(scriptPath)
- .setNmPrivateTokensPath(tokensPath)
- .setUser(appSubmitter)
- .setAppId(appId)
- .setContainerWorkDir(workDir)
- .setLocalDirs(dirsHandler.getLocalDirs())
- .setLogDirs(dirsHandler.getLogDirs())
- .setFilecacheDirs(new ArrayList<>())
- .setUserLocalDirs(new ArrayList<>())
- .setContainerLocalDirs(new ArrayList<>())
- .setContainerLogDirs(new ArrayList<>())
- .setUserFilecacheDirs(new ArrayList<>())
- .setApplicationLocalDirs(new ArrayList<>())
- .build());
+ ContainerStartContext.Builder ctxBuilder =
+ new ContainerStartContext.Builder()
+ .setContainer(container)
+ .setNmPrivateContainerScriptPath(scriptPath)
+ .setNmPrivateTokensPath(tokensPath)
+ .setUser(appSubmitter)
+ .setAppId(appId)
+ .setContainerWorkDir(workDir)
+ .setLocalDirs(dirsHandler.getLocalDirs())
+ .setLogDirs(dirsHandler.getLogDirs())
+ .setFilecacheDirs(new ArrayList<>())
+ .setUserLocalDirs(new ArrayList<>())
+ .setContainerLocalDirs(new ArrayList<>())
+ .setContainerLogDirs(new ArrayList<>())
+ .setUserFilecacheDirs(new ArrayList<>())
+ .setApplicationLocalDirs(new ArrayList<>());
+ if (https) {
+ ctxBuilder.setNmPrivateKeystorePath(keystorePath);
+ ctxBuilder.setNmPrivateTruststorePath(truststorePath);
+ }
+ int ret = mockExec.launchContainer(ctxBuilder.build());
assertEquals(0, ret);
- assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
- appSubmitter, cmd, appId, containerId,
- workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
- StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
- dirsHandler.getLocalDirs()),
- StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
- dirsHandler.getLogDirs()), "cgroups=none"),
- readMockParams());
-
+ if (https) {
+ assertEquals(Arrays.asList(
+ YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ appSubmitter, cmd, appId, containerId,
+ workDir.toString(), scriptPath.toUri().getPath(),
+ tokensPath.toUri().getPath(), "--https",
+ keystorePath.toUri().getPath(), truststorePath.toUri().getPath(),
+ pidFile.toString(),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ dirsHandler.getLocalDirs()),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ dirsHandler.getLogDirs()), "cgroups=none"),
+ readMockParams());
+ } else {
+ assertEquals(Arrays.asList(
+ YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ appSubmitter, cmd, appId, containerId,
+ workDir.toString(), scriptPath.toUri().getPath(),
+ tokensPath.toUri().getPath(), "--http", pidFile.toString(),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ dirsHandler.getLocalDirs()),
+ StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+ dirsHandler.getLogDirs()), "cgroups=none"),
+ readMockParams());
+ }
}
@Test (timeout = 5000)
public void testContainerLaunchWithPriority()
- throws IOException, ConfigurationException {
+ throws IOException, ConfigurationException, URISyntaxException {
// set the scheduler priority to make sure still works with nice -n prio
Configuration conf = new Configuration();
@@ -242,7 +280,7 @@ public void testContainerLaunchWithPriority()
assertEquals("third should be the priority", Integer.toString(2),
command.get(2));
- testContainerLaunch();
+ testContainerLaunchWithoutHTTPS();
}
@Test (timeout = 5000)
@@ -306,7 +344,7 @@ public void testStartLocalizer() throws IOException {
@Test
public void testContainerLaunchError()
- throws IOException, ContainerExecutionException {
+ throws IOException, ContainerExecutionException, URISyntaxException {
final String[] expecetedMessage = {"badcommand", "Exit code: 24"};
final String[] executor = {
@@ -410,7 +448,8 @@ public Object answer(InvocationOnMock invocationOnMock)
assertEquals(Arrays.asList(YarnConfiguration.
DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, appId, containerId,
- workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
+ workDir.toString(), "/bin/echo", "/dev/null", "--http",
+ pidFile.toString(),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
dirsHandler.getLocalDirs()),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
@@ -462,7 +501,7 @@ public void testContainerKill() throws IOException {
}
@Test
- public void testDeleteAsUser() throws IOException {
+ public void testDeleteAsUser() throws IOException, URISyntaxException {
String appSubmitter = "nobody";
String cmd = String.valueOf(
PrivilegedOperation.RunAsUserCommand.DELETE_AS_USER.getValue());
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 5714a1c1855..1f218d0942c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -25,6 +25,7 @@
import static org.mockito.Mockito.*;
import java.io.BufferedReader;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
@@ -55,9 +56,12 @@
import com.google.common.collect.Lists;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.test.GenericTestUtils;
@@ -108,11 +112,13 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.server.security.AMSecretKeys;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
@@ -124,6 +130,8 @@
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
public class TestContainerLaunch extends BaseContainerManagerTest {
@@ -2443,4 +2451,145 @@ public void testDistributedCacheDirs() throws Exception {
launch.getUserFilecacheDirs(localDirsForRead)),
StringUtils.join(",", ctx.getUserFilecacheDirs()));
}
+
+ @Test(timeout = 20000)
+ public void testFilesAndEnvWithoutHTTPS() throws Exception {
+ testFilesAndEnv(false);
+ }
+
+ @Test(timeout = 20000)
+ public void testFilesAndEnvWithHTTPS() throws Exception {
+ testFilesAndEnv(true);
+ }
+
+ private void testFilesAndEnv(boolean https) throws Exception {
+ // setup mocks
+ Dispatcher dispatcher = mock(Dispatcher.class);
+ EventHandler handler = mock(EventHandler.class);
+ when(dispatcher.getEventHandler()).thenReturn(handler);
+ ContainerExecutor containerExecutor = mock(ContainerExecutor.class);
+ doAnswer(new Answer() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ DataOutputStream dos = (DataOutputStream) args[0];
+ dos.writeBytes("script");
+ return null;
+ }
+ }).when(containerExecutor).writeLaunchEnv(
+ any(), any(), any(), any(), any(), any(), any());
+ Application app = mock(Application.class);
+ ApplicationId appId = mock(ApplicationId.class);
+ when(appId.toString()).thenReturn("1");
+ when(app.getAppId()).thenReturn(appId);
+ Container container = mock(Container.class);
+ ContainerId id = mock(ContainerId.class);
+ when(id.toString()).thenReturn("1");
+ when(container.getContainerId()).thenReturn(id);
+ when(container.getUser()).thenReturn("user");
+ ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
+ when(clc.getCommands()).thenReturn(Lists.newArrayList());
+ when(container.getLaunchContext()).thenReturn(clc);
+ Credentials credentials = mock(Credentials.class);
+ when(container.getCredentials()).thenReturn(credentials);
+ doAnswer(new Answer() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ DataOutputStream dos = (DataOutputStream) args[0];
+ dos.writeBytes("credentials");
+ return null;
+ }
+ }).when(credentials).writeTokenStorageToStream(any(DataOutputStream.class));
+ if (https) {
+ when(credentials.getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE))
+ .thenReturn("keystore".getBytes());
+ when(credentials.getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE_PASSWORD))
+ .thenReturn("keystore_password".getBytes());
+ when(credentials.getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE))
+ .thenReturn("truststore".getBytes());
+ when(credentials.getSecretKey(
+ AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE_PASSWORD))
+ .thenReturn("truststore_password".getBytes());
+ }
+
+ // call containerLaunch
+ ContainerLaunch containerLaunch = new ContainerLaunch(
+ distContext, conf, dispatcher,
+ containerExecutor, app, container, dirsHandler, containerManager);
+ containerLaunch.call();
+
+ // verify the nmPrivate paths and files
+ ArgumentCaptor cscArgument =
+ ArgumentCaptor.forClass(ContainerStartContext.class);
+ verify(containerExecutor, times(1)).launchContainer(cscArgument.capture());
+ ContainerStartContext csc = cscArgument.getValue();
+ Path nmPrivate = dirsHandler.getLocalPathForWrite(
+ ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR +
+ appId.toString() + Path.SEPARATOR + id.toString());
+ Assert.assertEquals(new Path(nmPrivate, ContainerLaunch.CONTAINER_SCRIPT),
+ csc.getNmPrivateContainerScriptPath());
+ Assert.assertEquals(new Path(nmPrivate,
+ String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
+ id.toString())), csc.getNmPrivateTokensPath());
+ Assert.assertEquals("script",
+ readStringFromPath(csc.getNmPrivateContainerScriptPath()));
+ Assert.assertEquals("credentials",
+ readStringFromPath(csc.getNmPrivateTokensPath()));
+ if (https) {
+ Assert.assertEquals(new Path(nmPrivate, ContainerLaunch.KEYSTORE_FILE),
+ csc.getNmPrivateKeystorePath());
+ Assert.assertEquals(new Path(nmPrivate, ContainerLaunch.TRUSTSTORE_FILE),
+ csc.getNmPrivateTruststorePath());
+ Assert.assertEquals("keystore",
+ readStringFromPath(csc.getNmPrivateKeystorePath()));
+ Assert.assertEquals("truststore",
+ readStringFromPath(csc.getNmPrivateTruststorePath()));
+ } else {
+ Assert.assertNull(csc.getNmPrivateKeystorePath());
+ Assert.assertNull(csc.getNmPrivateTruststorePath());
+ }
+
+ // verify env
+ ArgumentCaptor