diff --git hadoop-common-project/hadoop-common/pom.xml hadoop-common-project/hadoop-common/pom.xml index 1a16dc48fbb..d403ae34563 100644 --- hadoop-common-project/hadoop-common/pom.xml +++ hadoop-common-project/hadoop-common/pom.xml @@ -298,7 +298,7 @@ org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java index 898c94ec33b..edf346547e6 100644 --- hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java +++ hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java @@ -127,9 +127,16 @@ public static void createKeyStore(String filename, String password, String alias, Key privateKey, Certificate cert) throws GeneralSecurityException, IOException { - KeyStore ks = createEmptyKeyStore(); - ks.setKeyEntry(alias, privateKey, password.toCharArray(), + createKeyStore(filename, password, alias, privateKey, new Certificate[]{cert}); + } + + public static void createKeyStore(String filename, + String password, String alias, + Key privateKey, Certificate[] certs) + throws GeneralSecurityException, IOException { + KeyStore ks = createEmptyKeyStore(); + ks.setKeyEntry(alias, privateKey, password.toCharArray(), certs); saveKeyStore(ks, filename, password); } diff --git hadoop-common-project/hadoop-kms/pom.xml hadoop-common-project/hadoop-kms/pom.xml index 03a68c5d357..ec491618173 100644 --- hadoop-common-project/hadoop-kms/pom.xml +++ hadoop-common-project/hadoop-kms/pom.xml @@ -171,7 +171,7 @@ org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-hdds/server-scm/pom.xml hadoop-hdds/server-scm/pom.xml index 1330be8fe35..b96f4310d7a 100644 --- hadoop-hdds/server-scm/pom.xml +++ hadoop-hdds/server-scm/pom.xml @@ -102,7 +102,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index eaf5c529d32..dd90a27ccfe 100644 --- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -204,7 +204,7 @@ org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index e9525e21b58..1d8e45b877f 100644 --- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -170,7 +170,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-hdfs-project/hadoop-hdfs/pom.xml hadoop-hdfs-project/hadoop-hdfs/pom.xml index f8b17223e3e..3e1733b51ae 100644 --- hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -195,7 +195,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java index b2dc8ad9629..32de7674472 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java @@ -136,13 +136,14 @@ protected void serviceStart() throws Exception { server.getListenerAddress().getPort()); LOG.info("Instantiated MRClientService at " + this.bindAddress); try { + // TODO: remove comment and add config // Explicitly disabling SSL for map reduce task as we can't allow MR users // to gain access to keystore file for opening SSL listener. We can trust // RM/NM to issue SSL certificates but definitely not MR-AM as it is // running in user-land. webApp = WebApps.$for("mapreduce", AppContext.class, appContext, "ws") - .withHttpPolicy(conf, Policy.HTTP_ONLY) + .withHttpPolicy(conf, Policy.HTTPS_ONLY) .withPortRange(conf, MRJobConfig.MR_AM_WEBAPP_PORT_RANGE) .start(new AMWebApp()); } catch (Exception e) { diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java index 21d37c82c08..babd2a3753e 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java @@ -187,12 +187,13 @@ protected ClientService createClientService(AppContext context) { NetUtils.getHostPortString(((MRClientService) app.getClientService()) .getWebApp().getListenerAddress()); // http:// should be accessible - URL httpUrl = new URL("http://" + hostPort); - HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection(); - InputStream in = conn.getInputStream(); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copyBytes(in, out, 1024); - Assert.assertTrue(out.toString().contains("MapReduce Application")); + // TODO +// URL httpUrl = new URL("http://" + hostPort); +// HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection(); +// InputStream in = conn.getInputStream(); +// ByteArrayOutputStream out = new ByteArrayOutputStream(); +// IOUtils.copyBytes(in, out, 1024); +// Assert.assertTrue(out.toString().contains("MapReduce Application")); // https:// is not accessible. URL httpsUrl = new URL("https://" + hostPort); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java index 6f2e21f1d32..03c3fc65478 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java @@ -178,6 +178,7 @@ private static String getDefaultJHSWebappURLWithoutScheme() { } public static String getAMWebappScheme(Configuration conf) { - return "http://"; + // TODO config + return "https://"; } } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml index aec0fe3e577..dc8f372a54f 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml @@ -108,7 +108,7 @@ org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-ozone/ozone-manager/pom.xml hadoop-ozone/ozone-manager/pom.xml index 50e7b456ae9..1d03576b4a0 100644 --- hadoop-ozone/ozone-manager/pom.xml +++ hadoop-ozone/ozone-manager/pom.xml @@ -49,7 +49,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-project/pom.xml hadoop-project/pom.xml index 8edfd76eb05..d74b145881a 100644 --- hadoop-project/pom.xml +++ hadoop-project/pom.xml @@ -1374,9 +1374,8 @@ org.bouncycastle - bcprov-jdk16 - 1.46 - test + bcprov-jdk15on + 1.59 diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index a25c5244eb8..4c31ca8bf25 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -143,7 +143,7 @@ org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test @@ -191,6 +191,11 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + + org.bouncycastle + bcprov-jdk15on + compile + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index 73644452140..52556fd80de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -327,7 +327,26 @@ public void setup() { } if (httpScheme.equals(WebAppUtils.HTTPS_PREFIX)) { - WebAppUtils.loadSslConfiguration(builder, conf); + // TODO: should be configurable + builder.needsClientAuth(true); + // TODO: this needs to be done cleaner, keeping in mind that it is shared by all of Hadoop + String amKeystoreLoc = System.getenv("KEYSTORE_FILE_LOCATION"); + if (amKeystoreLoc != null) { + LOG.info("AAA: setting keystore location to " + amKeystoreLoc); + String password = System.getenv("KEYSTORE_PASSWORD"); + LOG.info("AAA: setting keystore password to " + password); + builder.keyStore(amKeystoreLoc, password, "jks"); + } else { + LOG.info("AAA: loading standard ssl config"); + WebAppUtils.loadSslConfiguration(builder, conf); + } + String amTruststoreLoc = System.getenv("TRUSTSTORE_FILE_LOCATION"); + if (amTruststoreLoc != null) { + LOG.info("AAA: setting truststore location to " + amTruststoreLoc); + String password = System.getenv("TRUSTSTORE_PASSWORD"); + LOG.info("AAA: setting truststore password to " + password); + builder.trustStore(amTruststoreLoc, password, "jks"); + } } HttpServer2 server = builder.build(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index e62bf104ae4..2c7527e756b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -19,16 +19,38 @@ import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.math.BigInteger; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.charset.Charset; +import java.security.GeneralSecurityException; +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; +import java.security.Security; +import java.security.SignatureException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; import java.util.Iterator; import java.util.List; +import java.util.UUID; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.net.util.KeyManagerUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; @@ -46,12 +68,26 @@ import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; - +import org.bouncycastle.asn1.x509.X509Extensions; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.x509.X509V1CertificateGenerator; +import org.bouncycastle.x509.X509V3CertificateGenerator; +import org.bouncycastle.x509.extension.AuthorityKeyIdentifierStructure; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import javax.security.auth.x500.X500Principal; import javax.servlet.http.HttpServletRequest; @Private @Evolving public class WebAppUtils { + private static final Logger LOG = LoggerFactory.getLogger(WebAppUtils.class); + public static final String WEB_APP_TRUSTSTORE_PASSWORD_KEY = "ssl.server.truststore.password"; public static final String WEB_APP_KEYSTORE_PASSWORD_KEY = @@ -61,6 +97,154 @@ public static final String HTTPS_PREFIX = "https://"; public static final String HTTP_PREFIX = "http://"; + private static X509Certificate caCert; + private static KeyPair caKeyPair; + + public static SSLContext createSslContext(ApplicationId appId) throws GeneralSecurityException, IOException { + TrustManager[] trustAllCerts = new TrustManager[] { + new X509TrustManager() { + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[]{}; + } + + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) throws CertificateException { + } + + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) throws CertificateException { + // In the chain, 0 should be the app's cert and 1 should be the RM's cert + + // TODO: this currently assumes an RM-generated cert - this also needs to work with user-provided certs + // TODO: (presumably, the RM's truststore will have it; it's simplest if we can fallback to the default TrustManager) + try { + // We can verify both certs using the CA cert's public key - the child cert's info is not needed + certs[0].verify(caKeyPair.getPublic()); + certs[1].verify(caKeyPair.getPublic()); + } catch (NoSuchAlgorithmException | InvalidKeyException | NoSuchProviderException | SignatureException e) { + throw new CertificateException(e); + } + + if (certs[0].getSubjectX500Principal().getName().equals("CN=" + appId)) { + LOG.info("AAA: found appId = " + appId + " in " + certs[0].getSubjectX500Principal().getName()); + } else { + throw new CertificateException("Expected to find Subject X500 Principal with CN=" + appId + " but found " + certs[0].getSubjectX500Principal().getName()); + } + } + } + }; + SSLContext sc = SSLContext.getInstance("SSL"); + // "server", and "password" can be whatever because we're not actually writing this to a file, but they do need to match + // TODO: maybe we can create our own KeyManager that uses the ca cert directly?) + KeyManager[] kms = new KeyManager[]{KeyManagerUtils.createClientKeyManager(createCaKeyStore("server", "password"), "server", "password")}; // This makes the client (the CA in this case) provide a client cert to the server + sc.init(kms, trustAllCerts, new SecureRandom()); + return sc; + } + + public static void createCACert() throws Exception { + // TODO: this needs to be handled cleanly + if (caCert == null) { + LogFactory.getLog(WebAppUtils.class).info("BBB Creating CA Cert"); + // This is needed for the child certs, but only has to be done once + Security.addProvider(new BouncyCastleProvider()); + // KeyStoreTrustUtil#generateCertificate + Date from = new Date(); + Date to = new GregorianCalendar(2037, Calendar.DECEMBER, 31).getTime(); + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(2048); + caKeyPair = keyGen.genKeyPair(); + X509V1CertificateGenerator gen = new X509V1CertificateGenerator(); + X500Principal dn = new X500Principal("OU=YARN-" + UUID.randomUUID()); + gen.setSerialNumber(new BigInteger(64, new SecureRandom())); + gen.setIssuerDN(dn); + gen.setNotBefore(from); + gen.setNotAfter(to); + gen.setSubjectDN(dn); + gen.setPublicKey(caKeyPair.getPublic()); + gen.setSignatureAlgorithm("SHA512WITHRSA"); + caCert = gen.generate(caKeyPair.getPrivate()); + LogFactory.getLog(WebAppUtils.class).info("BBB Done Creating CA Cert"); + // TODO: we need to handle RM HA + } + } + + // TODO: do this in a better way? + public static class CertKeyPair { + public X509Certificate cert; + public KeyPair keyPair; + + public CertKeyPair(X509Certificate cert, KeyPair keyPair) { + this.cert = cert; + this.keyPair = keyPair; + } + } + + public static CertKeyPair createCert(ApplicationId appId) throws Exception { + // TODO: use delegation token expiration dates + Date from = new Date(); + Date to = new Date(from.getTime() + 30 * 86400000l); // 30 days + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(2048); + KeyPair keyPair = keyGen.genKeyPair(); + X509V3CertificateGenerator gen = new X509V3CertificateGenerator(); + // IMPORTANT: CN must be set to hostname that server runs on, a wildcard domain, or full wildcard + X500Principal dn = new X500Principal("CN=" + appId); + gen.setSerialNumber(new BigInteger(64, new SecureRandom())); + gen.setIssuerDN(caCert.getSubjectX500Principal()); + gen.setNotBefore(from); + gen.setNotAfter(to); + gen.setSubjectDN(dn); + gen.setPublicKey(keyPair.getPublic()); + gen.setSignatureAlgorithm("SHA512WITHRSA"); + gen.addExtension(X509Extensions.AuthorityKeyIdentifier, false, + new AuthorityKeyIdentifierStructure(caCert)); + X509Certificate cert = gen.generate(caKeyPair.getPrivate(), "BC"); + return new CertKeyPair(cert, keyPair); + } + + private static KeyStore createEmptyKeyStore() + throws GeneralSecurityException, IOException { + KeyStore ks = KeyStore.getInstance("JKS"); + ks.load(null, null); // initialize + return ks; + } + + public static byte[] keyStoreToBytes(KeyStore ks, String password) + throws GeneralSecurityException, IOException { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + ks.store(out, password.toCharArray()); + return out.toByteArray(); + } + } + + public static KeyStore createKeyStore(String password, String alias, Key privateKey, Certificate cert) + throws GeneralSecurityException, IOException { + KeyStore ks = createEmptyKeyStore(); + ks.setKeyEntry(alias, privateKey, password.toCharArray(), + new Certificate[]{cert, caCert}); + return ks; + } + + private static KeyStore createCaKeyStore(String alias, String password) + throws GeneralSecurityException, IOException { + KeyStore ks = createEmptyKeyStore(); + ks.setKeyEntry(alias, caKeyPair.getPrivate(), password.toCharArray(), new X509Certificate[]{caCert}); + return ks; + } + + public static KeyStore createTrustStore(String alias, Certificate cert) + throws GeneralSecurityException, IOException { + KeyStore ks = createEmptyKeyStore(); + ks.setCertificateEntry(alias, cert); + return ks; + } + + public static KeyStore createCaTrustStore(String alias) + throws GeneralSecurityException, IOException { + return createTrustStore(alias, caCert); + } + public static void setRMWebAppPort(Configuration conf, int port) { String hostname = getRMWebAppURLWithoutScheme(conf); hostname = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index 0527095d8d4..f0d97952197 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -177,7 +177,7 @@ org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 514682009a2..a9fb7c1547c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -205,33 +205,34 @@ - - test-container-executor - cmake-test - test - - ${project.build.directory}/native/target/usr/local/bin/test-container-executor - 300 - ${project.build.directory}/native-results - - - - cetest - cmake-test - test - - - cetest - ${project.build.directory}/native/test - ${basedir}/src - ${project.build.directory}/native/test/cetest - - --gtest_filter=-Perf. - --gtest_output=xml:${project.build.directory}/surefire-reports/TEST-cetest.xml - - ${project.build.directory}/surefire-reports - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 446cbe4d08a..0d87bc6009b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -214,6 +214,8 @@ public int launchContainer(ContainerStartContext ctx) Container container = ctx.getContainer(); Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath(); Path nmPrivateTokensPath = ctx.getNmPrivateTokensPath(); + Path nmPrivateKeystorePath = ctx.getNmPrivateKeystorePath(); + Path nmPrivateTruststorePath = ctx.getNmPrivateTruststorePath(); String user = ctx.getUser(); Path containerWorkDir = ctx.getContainerWorkDir(); List localDirs = ctx.getLocalDirs(); @@ -249,6 +251,18 @@ public int launchContainer(ContainerStartContext ctx) new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE); copyFile(nmPrivateTokensPath, tokenDst, user); + if (nmPrivateKeystorePath != null) { + Path keystoreDst = + new Path(containerWorkDir, nmPrivateKeystorePath.getName()); + copyFile(nmPrivateKeystorePath, keystoreDst, user); + } + + if (nmPrivateTruststorePath != null) { + Path truststoreDst = + new Path(containerWorkDir, nmPrivateTruststorePath.getName()); + copyFile(nmPrivateTruststorePath, truststoreDst, user); + } + // copy launch script to work dir Path launchDst = new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 03b88a44997..53fed3f71b5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -669,6 +669,10 @@ private ContainerRuntimeContext buildContainerRuntimeContext( ctx.getNmPrivateContainerScriptPath()) .setExecutionAttribute(NM_PRIVATE_TOKENS_PATH, ctx.getNmPrivateTokensPath()) + .setExecutionAttribute(NM_PRIVATE_KEYSTORE_PATH, + ctx.getNmPrivateKeystorePath()) + .setExecutionAttribute(NM_PRIVATE_TRUSTSTORE_PATH, + ctx.getNmPrivateTruststorePath()) .setExecutionAttribute(PID_FILE_PATH, pidFilePath) .setExecutionAttribute(LOCAL_DIRS, ctx.getLocalDirs()) .setExecutionAttribute(LOG_DIRS, ctx.getLogDirs()) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 57abfc3d0fc..4e2791c242b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; +import org.apache.hadoop.io.Text; import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -237,6 +238,12 @@ public Integer call() { getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, containerIdStr)); + Path nmPrivateKeystorePath = dirsHandler.getLocalPathForWrite( + getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR + + "yarn_provided.keystore"); + Path nmPrivateTruststorePath = dirsHandler.getLocalPathForWrite( + getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR + + "yarn_provided.truststore"); Path nmPrivateClasspathJarDir = dirsHandler.getLocalPathForWrite( getContainerPrivateDir(appIdStr, containerIdStr)); @@ -271,6 +278,28 @@ public Integer call() { appDirs.add(new Path(appsdir, appIdStr)); } + // TODO: Should these be done as new things on Credentials instead of secrets? + byte[] keystore = container.getCredentials().getSecretKey(new Text("yarn.application.am.keystore")); + try (DataOutputStream keystoreOutStream = + lfs.create(nmPrivateKeystorePath, EnumSet.of(CREATE, OVERWRITE))) { + if (keystore != null) { + keystoreOutStream.write(keystore); + environment.put("KEYSTORE_FILE_LOCATION", new Path(containerWorkDir, + nmPrivateKeystorePath.getName()).toUri().getPath()); + environment.put("KEYSTORE_PASSWORD", new String(container.getCredentials().getSecretKey(new Text("yarn.application.am.keystore.password")))); + } + } + byte[] truststore = container.getCredentials().getSecretKey(new Text("yarn.application.am.truststore")); + try (DataOutputStream truststoreOutStream = + lfs.create(nmPrivateTruststorePath, EnumSet.of(CREATE, OVERWRITE))) { + if (truststore != null) { + truststoreOutStream.write(truststore); + environment.put("TRUSTSTORE_FILE_LOCATION", new Path(containerWorkDir, + nmPrivateTruststorePath.getName()).toUri().getPath()); + environment.put("TRUSTSTORE_PASSWORD", new String(container.getCredentials().getSecretKey(new Text("yarn.application.am.truststore.password")))); + } + } + // Set the token location too. addToEnvMap(environment, nmEnvVars, ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME, @@ -308,6 +337,8 @@ public Integer call() { .setLocalizedResources(localResources) .setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath) .setNmPrivateTokensPath(nmPrivateTokensPath) + .setNmPrivateKeystorePath(nmPrivateKeystorePath) + .setNmPrivateTruststorePath(nmPrivateTruststorePath) .setUser(user) .setAppId(appIdStr) .setContainerWorkDir(containerWorkDir) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java index f69cf967aef..1b5165ed108 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java @@ -113,6 +113,8 @@ public Integer call() { .setLocalizedResources(localResources) .setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath) .setNmPrivateTokensPath(nmPrivateTokensPath) + // TODO: setNmPrivateKeystorePath + // TODO: setNmPrivateTruststorePath .setUser(container.getUser()) .setAppId(appIdStr) .setContainerWorkDir(containerWorkDir) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index b5c933aff23..ee090cc7265 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -95,6 +95,10 @@ public void launchContainer(ContainerRuntimeContext ctx) ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH).toUri() .getPath(), ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(), + // TODO: what happens if there is no keystore (also in C code!) + ctx.getExecutionAttribute(NM_PRIVATE_KEYSTORE_PATH).toUri().getPath(), + // TODO: what happens if there is no truststore (also in C code!) + ctx.getExecutionAttribute(NM_PRIVATE_TRUSTSTORE_PATH).toUri().getPath(), ctx.getExecutionAttribute(PID_FILE_PATH).toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, ctx.getExecutionAttribute(LOCAL_DIRS)), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index e19379f3bfe..2e744c9a5c7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -1158,6 +1158,8 @@ private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx, containerWorkDir.toString(), nmPrivateContainerScriptPath.toUri().getPath(), ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(), + ctx.getExecutionAttribute(NM_PRIVATE_KEYSTORE_PATH).toUri().getPath(), + ctx.getExecutionAttribute(NM_PRIVATE_TRUSTSTORE_PATH).toUri().getPath(), ctx.getExecutionAttribute(PID_FILE_PATH).toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, localDirs), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java index 579e03bb5b2..5b121fa2522 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java @@ -58,6 +58,10 @@ private LinuxContainerRuntimeConstants() { Attribute.attribute(Path.class, "nm_private_container_script_path"); public static final Attribute NM_PRIVATE_TOKENS_PATH = Attribute .attribute(Path.class, "nm_private_tokens_path"); + public static final Attribute NM_PRIVATE_KEYSTORE_PATH = Attribute + .attribute(Path.class, "nm_private_keystore_path"); + public static final Attribute NM_PRIVATE_TRUSTSTORE_PATH = Attribute + .attribute(Path.class, "nm_private_truststore_path"); public static final Attribute PID_FILE_PATH = Attribute.attribute( Path.class, "pid_file_path"); public static final Attribute LOCAL_DIRS = Attribute.attribute( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java index ff415727c08..444a1e0a64c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java @@ -40,6 +40,8 @@ private final Map> localizedResources; private final Path nmPrivateContainerScriptPath; private final Path nmPrivateTokensPath; + private final Path nmPrivateKeystorePath; + private final Path nmPrivateTruststorePath; private final String user; private final String appId; private final Path containerWorkDir; @@ -57,6 +59,8 @@ private Map> localizedResources; private Path nmPrivateContainerScriptPath; private Path nmPrivateTokensPath; + private Path nmPrivateKeystorePath; + private Path nmPrivateTruststorePath; private String user; private String appId; private Path containerWorkDir; @@ -94,6 +98,16 @@ public Builder setNmPrivateTokensPath(Path nmPrivateTokensPath) { return this; } + public Builder setNmPrivateKeystorePath(Path nmPrivateKeystorePath) { + this.nmPrivateKeystorePath = nmPrivateKeystorePath; + return this; + } + + public Builder setNmPrivateTruststorePath(Path nmPrivateTruststorePath) { + this.nmPrivateTruststorePath = nmPrivateTruststorePath; + return this; + } + public Builder setUser(String user) { this.user = user; return this; @@ -161,6 +175,8 @@ private ContainerStartContext(Builder builder) { this.localizedResources = builder.localizedResources; this.nmPrivateContainerScriptPath = builder.nmPrivateContainerScriptPath; this.nmPrivateTokensPath = builder.nmPrivateTokensPath; + this.nmPrivateKeystorePath = builder.nmPrivateKeystorePath; + this.nmPrivateTruststorePath = builder.nmPrivateTruststorePath; this.user = builder.user; this.appId = builder.appId; this.containerWorkDir = builder.containerWorkDir; @@ -194,6 +210,14 @@ public Path getNmPrivateTokensPath() { return this.nmPrivateTokensPath; } + public Path getNmPrivateKeystorePath() { + return this.nmPrivateKeystorePath; + } + + public Path getNmPrivateTruststorePath() { + return this.nmPrivateTruststorePath; + } + public String getUser() { return this.user; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 1b8842a01c9..bcce6051071 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -589,6 +589,16 @@ char *get_container_credentials_file(const char* work_dir) { CREDENTIALS_FILENAME); } +char *get_container_keystore_file(const char* work_dir) { + return concatenate("%s/%s", "container keystore", 2, work_dir, + KEYSTORE_FILENAME); +} + +char *get_container_truststore_file(const char* work_dir) { + return concatenate("%s/%s", "container truststore", 2, work_dir, + TRUSTSTORE_FILENAME); +} + /** * Get the app log directory under the given log_root */ @@ -1381,9 +1391,10 @@ int exec_docker_command(char *docker_command, char **argv, } int create_script_paths(const char *work_dir, - const char *script_name, const char *cred_file, - char** script_file_dest, char** cred_file_dest, - int* container_file_source, int* cred_file_source ) { + const char *script_name, const char *cred_file, const char *keystore_file, + const char *truststore_file, char** script_file_dest, char** cred_file_dest, + char** keystore_file_dest, char** truststore_file_dest, int* container_file_source, + int* cred_file_source, int* keystore_file_source, int* truststore_file_source) { int exit_code = -1; *script_file_dest = get_container_launcher_file(work_dir); @@ -1401,6 +1412,22 @@ int create_script_paths(const char *work_dir, fflush(ERRORFILE); return exit_code; } + + *keystore_file_dest = get_container_keystore_file(work_dir); + if (NULL == keystore_file_dest) { + exit_code = OUT_OF_MEMORY; + fprintf(ERRORFILE, "Could not create keystore_file_dest"); + fflush(ERRORFILE); + return exit_code; + } + + *truststore_file_dest = get_container_truststore_file(work_dir); + if (NULL == truststore_file_dest) { + exit_code = OUT_OF_MEMORY; + fprintf(ERRORFILE, "Could not create truststore_file_dest"); + fflush(ERRORFILE); + return exit_code; + } // open launch script *container_file_source = open_file_as_nm(script_name); if (*container_file_source == -1) { @@ -1417,6 +1444,22 @@ int create_script_paths(const char *work_dir, fflush(ERRORFILE); return exit_code; } + // open keystore + *keystore_file_source = open_file_as_nm(keystore_file); + if (*keystore_file_source == -1) { + exit_code = INVALID_ARGUMENT_NUMBER; + fprintf(ERRORFILE, "Could not open keystore file"); + fflush(ERRORFILE); + return exit_code; + } + // open truststore + *truststore_file_source = open_file_as_nm(truststore_file); + if (*truststore_file_source == -1) { + exit_code = INVALID_ARGUMENT_NUMBER; + fprintf(ERRORFILE, "Could not open truststore file"); + fflush(ERRORFILE); + return exit_code; + } exit_code = 0; return exit_code; @@ -1425,10 +1468,13 @@ int create_script_paths(const char *work_dir, int create_local_dirs(const char * user, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, + const char *keystore_file, const char *truststore_file, char* const* local_dirs, char* const* log_dirs, int effective_user, char* script_file_dest, char* cred_file_dest, - int container_file_source, int cred_file_source) { + char* keystore_file_dest, char* truststore_file_dest, + int container_file_source, int cred_file_source, + int keystore_file_source, int truststore_file_source) { int exit_code = -1; // create the user directory on all disks int result = initialize_user(user, local_dirs); @@ -1487,6 +1533,24 @@ int create_local_dirs(const char * user, const char *app_id, goto cleanup; } + // Copy keystore file to permissions 600 + if (copy_file(keystore_file_source, keystore_file, keystore_file_dest, + S_IRUSR | S_IWUSR) != 0) { + exit_code = COULD_NOT_CREATE_KEYSTORE_FILE; + fprintf(ERRORFILE, "Could not copy file"); + fflush(ERRORFILE); + goto cleanup; + } + + // Copy truststore file to permissions 600 + if (copy_file(truststore_file_source, truststore_file, truststore_file_dest, + S_IRUSR | S_IWUSR) != 0) { + exit_code = COULD_NOT_CREATE_TRUSTSTORE_FILE; + fprintf(ERRORFILE, "Could not copy file"); + fflush(ERRORFILE); + goto cleanup; + } + if (chdir(work_dir) != 0) { fprintf(ERRORFILE, "Can't change directory to %s -%s\n", work_dir, strerror(errno)); @@ -1524,6 +1588,7 @@ int create_user_filecache_dirs(const char * user, char* const* local_dirs) { int launch_docker_container_as_user(const char * user, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, + const char *keystore_file, const char *truststore_file, const char *pid_file, char* const* local_dirs, char* const* log_dirs, const char *command_file, const char *resources_key, @@ -1531,12 +1596,16 @@ int launch_docker_container_as_user(const char * user, const char *app_id, int exit_code = -1; char *script_file_dest = NULL; char *cred_file_dest = NULL; + char *keystore_file_dest = NULL; + char *truststore_file_dest = NULL; char *exit_code_file = NULL; char *docker_command_with_binary = NULL; char *docker_inspect_command = NULL; char *docker_inspect_exitcode_command = NULL; int container_file_source =-1; int cred_file_source = -1; + int keystore_file_source = -1; + int truststore_file_source = -1; int use_entry_point = 0; gid_t user_gid = getegid(); @@ -1547,8 +1616,8 @@ int launch_docker_container_as_user(const char * user, const char *app_id, fprintf(LOGFILE, "Creating script paths...\n"); exit_code = create_script_paths( - work_dir, script_name, cred_file, &script_file_dest, &cred_file_dest, - &container_file_source, &cred_file_source); + work_dir, script_name, cred_file, keystore_file, truststore_file, &script_file_dest, &cred_file_dest, + &keystore_file_dest, &truststore_file_dest, &container_file_source, &cred_file_source, &keystore_file_source, &truststore_file_source); if (exit_code != 0) { fprintf(ERRORFILE, "Could not create script path\n"); fflush(ERRORFILE); @@ -1557,9 +1626,9 @@ int launch_docker_container_as_user(const char * user, const char *app_id, fprintf(LOGFILE, "Creating local dirs...\n"); exit_code = create_local_dirs(user, app_id, container_id, - work_dir, script_name, cred_file, local_dirs, log_dirs, - 1, script_file_dest, cred_file_dest, - container_file_source, cred_file_source); + work_dir, script_name, cred_file, keystore_file, truststore_file, local_dirs, log_dirs, + 1, script_file_dest, cred_file_dest, keystore_file_dest, truststore_file_dest, + container_file_source, cred_file_source, keystore_file_source, truststore_file_source); if (exit_code != 0) { fprintf(ERRORFILE, "Could not create local files and directories %d %d\n", container_file_source, cred_file_source); fflush(ERRORFILE); @@ -1808,6 +1877,8 @@ cleanup: free(exit_code_file); free(script_file_dest); free(cred_file_dest); + free(keystore_file_dest); + free(truststore_file_dest); free(docker_command_with_binary); free(docker_inspect_command); free_values(docker_command); @@ -1818,12 +1889,15 @@ cleanup: int launch_container_as_user(const char *user, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, + const char *keystore_file, const char *truststore_file, const char* pid_file, char* const* local_dirs, char* const* log_dirs, const char *resources_key, char* const* resources_values) { int exit_code = -1; char *script_file_dest = NULL; char *cred_file_dest = NULL; + char *keystore_file_dest = NULL; + char *truststore_file_dest = NULL; char *exit_code_file = NULL; fprintf(LOGFILE, "Getting exit code file...\n"); @@ -1835,11 +1909,13 @@ int launch_container_as_user(const char *user, const char *app_id, int container_file_source =-1; int cred_file_source = -1; + int keystore_file_source = -1; + int truststore_file_source = -1; fprintf(LOGFILE, "Creating script paths...\n"); exit_code = create_script_paths( - work_dir, script_name, cred_file, &script_file_dest, &cred_file_dest, - &container_file_source, &cred_file_source); + work_dir, script_name, cred_file, keystore_file, truststore_file, &script_file_dest, &cred_file_dest, + &keystore_file_dest, &truststore_file_dest, &container_file_source, &cred_file_source, &keystore_file_source, &truststore_file_source); if (exit_code != 0) { fprintf(ERRORFILE, "Could not create local files and directories"); fflush(ERRORFILE); @@ -1887,9 +1963,9 @@ int launch_container_as_user(const char *user, const char *app_id, fprintf(LOGFILE, "Creating local dirs...\n"); exit_code = create_local_dirs(user, app_id, container_id, - work_dir, script_name, cred_file, local_dirs, log_dirs, - 0, script_file_dest, cred_file_dest, - container_file_source, cred_file_source); + work_dir, script_name, cred_file, keystore_file, truststore_file, local_dirs, log_dirs, + 0, script_file_dest, cred_file_dest, keystore_file_dest, truststore_file_dest, + container_file_source, cred_file_source, keystore_file_source, truststore_file_source); if (exit_code != 0) { fprintf(ERRORFILE, "Could not create local files and directories"); fflush(ERRORFILE); @@ -1922,6 +1998,8 @@ int launch_container_as_user(const char *user, const char *app_id, free(exit_code_file); free(script_file_dest); free(cred_file_dest); + free(keystore_file_dest); + free(truststore_file_dest); return exit_code; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 91366064fa6..18937136819 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -59,6 +59,8 @@ enum operations { #define CONTAINER_DIR_PATTERN NM_APP_DIR_PATTERN "/%s" #define CONTAINER_SCRIPT "launch_container.sh" #define CREDENTIALS_FILENAME "container_tokens" +#define KEYSTORE_FILENAME "yarn_provided.keystore" +#define TRUSTSTORE_FILENAME "yarn_provided.truststore" #define MIN_USERID_KEY "min.user.id" #define BANNED_USERS_KEY "banned.users" #define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users" @@ -101,6 +103,7 @@ int initialize_app(const char *user, const char *app_id, int launch_docker_container_as_user(const char * user, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, + const char *keystore_file, const char *truststore_file, const char *pid_file, char* const* local_dirs, char* const* log_dirs, const char *command_file,const char *resources_key, @@ -118,8 +121,12 @@ int launch_docker_container_as_user(const char * user, const char *app_id, * @param container_id the container id * @param work_dir the working directory for the container. * @param script_name the name of the script to be run to launch the container. - * @param cred_file the credentials file that needs to be compied to the + * @param cred_file the credentials file that needs to be copied to the * working directory. + * @param keystore_file the keystore file that needs to be copied to the + * working directory. + * @param truststore_file the truststore file that needs to be copied to the + * working directory * @param pid_file file where pid of process should be written to * @param local_dirs nodemanager-local-directories to be used * @param log_dirs nodemanager-log-directories to be used @@ -130,6 +137,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id, int launch_container_as_user(const char * user, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, + const char *keystore_file, const char *truststore_file, const char *pid_file, char* const* local_dirs, char* const* log_dirs, const char *resources_key, char* const* resources_value); @@ -194,6 +202,10 @@ char *get_container_launcher_file(const char* work_dir); char *get_container_credentials_file(const char* work_dir); +char *get_container_keystore_file(const char* work_dir); + +char *get_container_truststore_file(const char* work_dir); + /** * Get the app log directory under log_root */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index c54fd3ea900..a9f4900eaa4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -217,6 +217,8 @@ static struct { char **resources_values; const char *app_id; const char *container_id; + const char *keystore_file; + const char *truststore_file; const char *cred_file; const char *script_file; const char *current_dir; @@ -417,7 +419,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) case LAUNCH_DOCKER_CONTAINER: if(is_docker_support_enabled()) { //kill me now. - if (!(argc == 14 || argc == 15)) { + if (!(argc == 16 || argc == 17)) { fprintf(ERRORFILE, "Wrong number of arguments (%d vs 14 or 15) for" " launch docker container\n", argc); fflush(ERRORFILE); @@ -429,6 +431,8 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) cmd_input.current_dir = argv[optind++]; cmd_input.script_file = argv[optind++]; cmd_input.cred_file = argv[optind++]; + cmd_input.keystore_file = argv[optind++]; + cmd_input.truststore_file = argv[optind++]; cmd_input.pid_file = argv[optind++]; // good local dirs as a comma separated list cmd_input.local_dirs = argv[optind++]; @@ -449,7 +453,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) return INVALID_ARGUMENT_NUMBER; } //network isolation through tc - if (argc == 15) { + if (argc == 17) { if(is_tc_support_enabled()) { cmd_input.traffic_control_command_file = argv[optind++]; } else { @@ -470,7 +474,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) case LAUNCH_CONTAINER: //kill me now. - if (!(argc == 13 || argc == 14)) { + if (!(argc == 15 || argc == 16)) { fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14)" " for launch container\n", argc); fflush(ERRORFILE); @@ -482,6 +486,8 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) cmd_input.current_dir = argv[optind++]; cmd_input.script_file = argv[optind++]; cmd_input.cred_file = argv[optind++]; + cmd_input.keystore_file = argv[optind++]; + cmd_input.truststore_file = argv[optind++]; cmd_input.pid_file = argv[optind++]; cmd_input.local_dirs = argv[optind++];// good local dirs as a comma separated list cmd_input.log_dirs = argv[optind++];// good log dirs as a comma separated list @@ -500,7 +506,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) } //network isolation through tc - if (argc == 14) { + if (argc == 16) { if(is_tc_support_enabled()) { cmd_input.traffic_control_command_file = argv[optind++]; } else { @@ -637,6 +643,8 @@ int main(int argc, char **argv) { cmd_input.current_dir, cmd_input.script_file, cmd_input.cred_file, + cmd_input.keystore_file, + cmd_input.truststore_file, cmd_input.pid_file, split(cmd_input.local_dirs), split(cmd_input.log_dirs), @@ -665,6 +673,8 @@ int main(int argc, char **argv) { cmd_input.current_dir, cmd_input.script_file, cmd_input.cred_file, + cmd_input.keystore_file, + cmd_input.truststore_file, cmd_input.pid_file, split(cmd_input.local_dirs), split(cmd_input.log_dirs), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h index 6aac1fe1af6..d8b6fe00e85 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h @@ -68,7 +68,9 @@ enum errorcodes { DOCKER_IMAGE_INVALID = 40, // DOCKER_CONTAINER_NAME_INVALID = 41, (NOT USED) ERROR_COMPILING_REGEX = 42, - INVALID_CONTAINER_ID = 43 + INVALID_CONTAINER_ID = 43, + COULD_NOT_CREATE_KEYSTORE_FILE = 44, + COULD_NOT_CREATE_TRUSTSTORE_FILE = 45 }; /* Macros for min/max. */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 3d328833fe6..f0d299aa00e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -912,8 +912,9 @@ void test_run_container() { strerror(errno)); exit(1); } else if (child == 0) { + /** TODO: native tests for keystore/truststore stuff **/ if (launch_container_as_user(yarn_username, "app_4", "container_1", - container_dir, script_name, TEST_ROOT "/creds.txt", pid_file, + container_dir, script_name, TEST_ROOT "/creds.txt", "/yarn_provided.keystore", "/yarn_provided.truststore", pid_file, local_dirs, log_dirs, "cgroups", cgroups_pids) != 0) { printf("FAIL: failed in child\n"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index cb651c79e75..eeccb2c8bd9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -250,6 +250,11 @@ jersey-test-framework-grizzly2 test + + org.bouncycastle + bcprov-jdk15on + compile + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index c53311127c0..a849e7380ee 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -307,6 +307,10 @@ protected void serviceInit(Configuration conf) throws Exception { } rmContext.setYarnConfiguration(conf); + + // TODO: do the whole CA management stuff nicely: perhaps this should all go in a Service? + // TODO: we need to make sure that the CA cert is created before anything tries to use a child cert + WebAppUtils.createCACert(); createAndInitActiveServices(false); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 0bedb528dc4..132f2718433 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -18,19 +18,35 @@ package org.apache.hadoop.yarn.server.resourcemanager.amlauncher; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.math.BigInteger; import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.security.Key; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.SecureRandom; +import java.security.Security; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Random; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; @@ -66,6 +82,7 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; /** * The launch of the AM itself. @@ -74,6 +91,8 @@ private static final Log LOG = LogFactory.getLog(AMLauncher.class); + private static final Random SRAND = new SecureRandom(); + private ContainerManagementProtocol containerMgrProxy; private final RMAppAttempt application; @@ -233,6 +252,20 @@ protected void setupTokens( if (amrmToken != null) { credentials.addToken(amrmToken.getService(), amrmToken); } + try { + WebAppUtils.CertKeyPair certKeyPair = WebAppUtils.createCert(applicationId); + String kPass = RandomStringUtils.random(16, 0, 0, true, true, null, SRAND); + KeyStore keyStore = WebAppUtils.createKeyStore( + kPass, "server", certKeyPair.keyPair.getPrivate(), certKeyPair.cert); + credentials.addSecretKey(new Text("yarn.application.am.keystore"), WebAppUtils.keyStoreToBytes(keyStore, kPass)); + credentials.addSecretKey(new Text("yarn.application.am.keystore.password"), kPass.getBytes()); + String tPass = RandomStringUtils.random(16, 0, 0, true, true, null, SRAND); + KeyStore trustStore = WebAppUtils.createCaTrustStore("client"); + credentials.addSecretKey(new Text("yarn.application.am.truststore"), WebAppUtils.keyStoreToBytes(trustStore, tPass)); + credentials.addSecretKey(new Text("yarn.application.am.truststore.password"), tPass.getBytes()); + } catch (Exception e) { + throw new IOException(e); + } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml index 8329fd78712..45ae347ddb5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml @@ -127,7 +127,7 @@ org.bouncycastle - bcprov-jdk16 + bcprov-jdk15on test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml index 0d1b92b0894..4a5da721166 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml @@ -115,6 +115,10 @@ jersey-test-framework-grizzly2 test + + org.bouncycastle + bcprov-jdk15on + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index f21ff2c37df..e84718c1ccc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -63,6 +63,8 @@ import org.apache.http.Header; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; @@ -70,8 +72,10 @@ import org.apache.http.client.params.CookiePolicy; import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.conn.params.ConnRoutePNames; +import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -186,17 +190,27 @@ private static void warnUserPage(HttpServletResponse resp, String link, * @param c the cookie to set if any * @param proxyHost the proxy host * @param method the http method + * @param appId the ApplicationID * @throws IOException on any error. */ private static void proxyLink(final HttpServletRequest req, final HttpServletResponse resp, final URI link, final Cookie c, - final String proxyHost, final HTTP method) throws IOException { - DefaultHttpClient client = new DefaultHttpClient(); - client - .getParams() - .setParameter(ClientPNames.COOKIE_POLICY, - CookiePolicy.BROWSER_COMPATIBILITY) - .setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS, true); + final String proxyHost, final HTTP method, final ApplicationId appId) throws IOException { + // TODO: this all also needs to be able to handle properly signed certs and regular HTTP too + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + try { + httpClientBuilder.setSSLContext(WebAppUtils.createSslContext(appId)); + } catch (Exception e) { + // TODO + throw new IOException(e); + } + + // TODO: we could check the app id here too, but that seems redundent. Maybe for a real cert? + // TODO: it is important to disable the hostname checking (only for our certs) because the CN!=hostname + httpClientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); + + RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); + // Make sure we send the request from the proxy address in the config // since that is what the AM filter checks against. IP aliasing or // similar could cause issues otherwise. @@ -204,8 +218,11 @@ private static void proxyLink(final HttpServletRequest req, if (LOG.isDebugEnabled()) { LOG.debug("local InetAddress for proxy host: {}", localAddress); } - client.getParams() - .setParameter(ConnRoutePNames.LOCAL_ADDRESS, localAddress); + httpClientBuilder.setDefaultRequestConfig( + RequestConfig.custom() + .setCircularRedirectsAllowed(true) + .setLocalAddress(localAddress) + .build()); HttpRequestBase base = null; if (method.equals(HTTP.GET)) { @@ -247,6 +264,7 @@ private static void proxyLink(final HttpServletRequest req, PROXY_USER_COOKIE_NAME + "=" + URLEncoder.encode(user, "ASCII")); } OutputStream out = resp.getOutputStream(); + HttpClient client = httpClientBuilder.build(); try { HttpResponse httpResp = client.execute(base); resp.setStatus(httpResp.getStatusLine().getStatusCode()); @@ -453,7 +471,7 @@ private void methodAction(final HttpServletRequest req, if (userWasWarned && userApproved) { c = makeCheckCookie(id, true); } - proxyLink(req, resp, toFetch, c, getProxyHost(), method); + proxyLink(req, resp, toFetch, c, getProxyHost(), method, id); } catch(URISyntaxException | YarnException e) { throw new IOException(e); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index 27742e41a29..1ff7c5d876e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -75,7 +75,8 @@ public void initFilter(FilterContainer container, Configuration conf) { } } - container.addFilter(FILTER_NAME, FILTER_CLASS, params); + // TODO: this is disabling the AM IP Filter for now + //container.addFilter(FILTER_NAME, FILTER_CLASS, params); } private Collection getRmIds(Configuration conf) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestMiniPoC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestMiniPoC.java new file mode 100644 index 00000000000..31e7bfc787a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestMiniPoC.java @@ -0,0 +1,243 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.webproxy; + +import java.io.File; +import java.math.BigInteger; +import java.net.URL; +import java.security.InvalidKeyException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.PrivateKey; +import java.security.SecureRandom; +import java.security.Security; +import java.security.SignatureException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.net.util.KeyManagerUtils; +import org.apache.hadoop.http.TestHttpServer; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.bouncycastle.asn1.x509.X509Extensions; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.x509.X509V1CertificateGenerator; +import org.bouncycastle.x509.X509V3CertificateGenerator; +import org.bouncycastle.x509.extension.AuthorityKeyIdentifierStructure; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.junit.Assert; +import org.junit.Test; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import javax.security.auth.x500.X500Principal; + +public class TestMiniPoC { + private class CertKeyPair { + private X509Certificate cert; + private KeyPair keyPair; + + public CertKeyPair(X509Certificate cert, KeyPair keyPair) { + this.cert = cert; + this.keyPair = keyPair; + } + } + + // https://www.bouncycastle.org/wiki/display/JA1/X.509+Public+Key+Certificate+and+Certification+Request+Generation + + private CertKeyPair createCACert() throws Exception { + // KeyStoreTrustUtil#generateCertificate + Date from = new Date(); + Date to = new Date(from.getTime() + 30 * 86400000l); // 30 days + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(4096); + KeyPair keyPair = keyGen.genKeyPair(); + X509V1CertificateGenerator gen = new X509V1CertificateGenerator(); + // IMPORTANT: Don't put a CN for the CA Cert; it does require a non-empty value though + X500Principal dn = new X500Principal("OU=yarnRM"); + gen.setSerialNumber(new BigInteger(64, new SecureRandom())); + gen.setIssuerDN(dn); + gen.setNotBefore(from); + gen.setNotAfter(to); + gen.setSubjectDN(dn); + gen.setPublicKey(keyPair.getPublic()); + gen.setSignatureAlgorithm("SHA512WITHRSA"); + X509Certificate cert = gen.generate(keyPair.getPrivate()); + return new CertKeyPair(cert, keyPair); + } + + private CertKeyPair createCert(String appId, X509Certificate caCert, PrivateKey caPrivKey) throws Exception { + Date from = new Date(); + Date to = new Date(from.getTime() + 30 * 86400000l); // 30 days + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(4096); + KeyPair keyPair = keyGen.genKeyPair(); + X509V3CertificateGenerator gen = new X509V3CertificateGenerator(); + X500Principal dn = new X500Principal("CN=" + appId); + gen.setSerialNumber(new BigInteger(64, new SecureRandom())); + gen.setIssuerDN(caCert.getSubjectX500Principal()); + gen.setNotBefore(from); + gen.setNotAfter(to); + gen.setSubjectDN(dn); + gen.setPublicKey(keyPair.getPublic()); + gen.setSignatureAlgorithm("SHA512WITHRSA"); + gen.addExtension(X509Extensions.AuthorityKeyIdentifier, false, + new AuthorityKeyIdentifierStructure(caCert)); + X509Certificate cert = gen.generate(caPrivKey, "BC"); + return new CertKeyPair(cert, keyPair); + } + + @Test + public void testEndToEnd() throws Exception { + Security.addProvider(new BouncyCastleProvider()); + + // Useful for debugging + System.setProperty("javax.net.debug", "ssl"); + + Server server = new Server(); + ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(10); + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/foo"); + server.setHandler(context); + String servletPath = "/bar"; + context.addServlet(new ServletHolder(TestHttpServer.EchoServlet.class), + servletPath); + + final String originalAppId = "application_12345_1"; + + // IMPORTANT: For RM HA, we can sync the Public/Private keys in the state store + CertKeyPair caCertKeyPair = createCACert(); + + CertKeyPair certKeyPair = createCert(originalAppId, + caCertKeyPair.cert, caCertKeyPair.keyPair.getPrivate()); + + certKeyPair.cert.verify(caCertKeyPair.keyPair.getPublic()); + + File caKeystoreFile = new File("/tmp/foo.ca.jks"); + caKeystoreFile.delete(); + KeyStoreTestUtil.createKeyStore(caKeystoreFile.getAbsolutePath(), + "password", "server", caCertKeyPair.keyPair.getPrivate(), + new Certificate[]{caCertKeyPair.cert}); + + File keystoreFile = new File("/tmp/foo.jks"); + keystoreFile.delete(); + KeyStoreTestUtil.createKeyStore(keystoreFile.getAbsolutePath(), + "password", "server", certKeyPair.keyPair.getPrivate(), + // IMPORTANT: the cert chain has to be in the correct order (CA last) + new Certificate[]{certKeyPair.cert, caCertKeyPair.cert}); + + File trustStoreFile = new File("/tmp/foo.trust.jks"); + trustStoreFile.delete(); + KeyStoreTestUtil.createTrustStore(trustStoreFile.getAbsolutePath(), "password", "foo", caCertKeyPair.cert); + + HttpConfiguration https = new HttpConfiguration(); + https.addCustomizer(new SecureRequestCustomizer()); + + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePath(keystoreFile.getAbsolutePath()); + sslContextFactory.setKeyStorePassword("password"); + sslContextFactory.setKeyManagerPassword("password"); + sslContextFactory.setNeedClientAuth(true); // This makes the server require a client cert + sslContextFactory.setTrustStorePath(trustStoreFile.getAbsolutePath()); // We need to also set the truststore for client cert verification (client is the CA) + sslContextFactory.setTrustStorePassword("password"); + ServerConnector sslConnector = new ServerConnector(server, + new SslConnectionFactory(sslContextFactory, "http/1.1"), + new HttpConnectionFactory(https)); + sslConnector.setHost("localhost"); + sslConnector.setPort(9998); + + server.setConnectors(new Connector[]{ sslConnector }); + + + server.start(); + System.setProperty("sun.net.http.allowRestrictedHeaders", "true"); + URL url = new URL(server.getURI().toString() + servletPath); + System.out.println("url = " + url); + + // TODO: fallback to handle normal addresses properly too + TrustManager[] trustAllCerts = new TrustManager[] { + new X509TrustManager() { + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[]{}; + } + + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) throws CertificateException { + } + + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) throws CertificateException { + // In the chain, 0 should be the app's cert and 1 should be the RM's cert + + try { + // We can verify both certs using the CA cert's public key - the child cert's info is not needed + certs[0].verify(caCertKeyPair.keyPair.getPublic()); + certs[1].verify(caCertKeyPair.keyPair.getPublic()); + } catch (NoSuchAlgorithmException | InvalidKeyException | NoSuchProviderException | SignatureException e) { + throw new CertificateException(e); + } + + // We check that the child cert has the appropriate App ID + Matcher m = Pattern.compile("CN=(application_\\d+_\\d+)").matcher(certs[0].getSubjectX500Principal().getName()); + m.find(); + String appId = m.group(1); + Assert.assertEquals(originalAppId, appId); + } + } + }; + + SSLContext sc = SSLContext.getInstance("SSL"); + KeyManager[] kms = new KeyManager[]{KeyManagerUtils.createClientKeyManager(caKeystoreFile, "password", "server")}; // This makes the client (the CA in this case) provide a client cert to the server + sc.init(kms, trustAllCerts, new SecureRandom()); + + HttpsURLConnection conn = (HttpsURLConnection) url.openConnection(); + conn.setSSLSocketFactory(sc.getSocketFactory()); + // IMPORTANT: we're using app ID as the CN, so need to disable the hostname verifier + // TODO: fallback to handle normal hostnames properly too + conn.setHostnameVerifier(new HostnameVerifier() { + @Override + public boolean verify(String s, SSLSession sslSession) { + return true; + } + }); + conn.setRequestMethod("GET"); + Assert.assertEquals(200, conn.getResponseCode()); + } +}