diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index f8ed4d5..77d7c77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -209,6 +209,9 @@ definitions: queue: type: string description: The YARN queue that this application should be submitted to. + KerberosPrincipal: + description: The Kerberos Principal of the application + $ref: '#/definitions/KerberosPrincipal' Resource: description: Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise. @@ -420,3 +423,12 @@ definitions: format: int32 description: An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information. + KerberosPrincipal: + description: The kerberos principal of the application. + properties: + principal_name: + type: string + description: The principal name of the application. + keytab_location: + type: string + description: The location of the keytab on a network accessible system such as hdfs. If location is not specified, it will look for the keytab under the system's configuration directory. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java index 4b7e59b..dd495cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java @@ -62,6 +62,7 @@ private ApplicationState state = null; private Map quicklinks = new HashMap<>(); private String queue = null; + private KerberosPrincipal kerberosPrincipal = null; /** * A unique application name. @@ -399,6 +400,24 @@ public void setQueue(String queue) { this.queue = queue; } + public Application kerberosPrincipal(KerberosPrincipal kerberosPrincipal) { + this.kerberosPrincipal = kerberosPrincipal; + return this; + } + + /** + * The Kerberos Principal of the application + * @return kerberosPrincipal + **/ + @ApiModelProperty(example = "null", value = "The Kerberos Principal of the application") + public KerberosPrincipal getKerberosPrincipal() { + return kerberosPrincipal; + } + + public void setKerberosPrincipal(KerberosPrincipal kerberosPrincipal) { + this.kerberosPrincipal = kerberosPrincipal; + } + @Override public boolean equals(java.lang.Object o) { if (this == o) { @@ -446,6 +465,7 @@ public String toString() { sb.append(" quicklinks: ").append(toIndentedString(quicklinks)) .append("\n"); sb.append(" queue: ").append(toIndentedString(queue)).append("\n"); + sb.append(" kerberosPrincipal: ").append(toIndentedString(kerberosPrincipal)).append("\n"); sb.append("}"); return sb.toString(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/KerberosPrincipal.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/KerberosPrincipal.java new file mode 100644 index 0000000..af06ef8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/KerberosPrincipal.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.resource; + +import java.io.Serializable; +import java.util.Objects; + +import com.google.gson.annotations.SerializedName; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +/** + * The kerberos principal of the application. + */ +@ApiModel(description = "The kerberos principal of the application.") +@javax.annotation.Generated(value = "io.swagger.codegen.languages.JavaClientCodegen", date = "2017-06-01T14:55:26.095-07:00") +public class KerberosPrincipal implements Serializable{ + + private static final long serialVersionUID = 3608921653111099035L; + + + @SerializedName("principal_name") private String principalName = null; + + @SerializedName("keytab_location") private String keytabLocation = null; + + public KerberosPrincipal principalName(String principalName) { + this.principalName = principalName; + return this; + } + + /** + * The principal name of the appliction. + * + * @return principalName + **/ + @ApiModelProperty(example = "null", value = "The principal name of the appliction.") + public String getPrincipalName() { + return principalName; + } + + public void setPrincipalName(String principalName) { + this.principalName = principalName; + } + + public KerberosPrincipal keytabLocation(String keytabLocation) { + this.keytabLocation = keytabLocation; + return this; + } + + /** + * The location of the keytab on a network accessible system such as hdfs. If location is not specified, it will look for the keytab under the system's configuration directory. + * + * @return keytabLocation + **/ + @ApiModelProperty(example = "null", value = "The location of the keytab on a network accessible system such as hdfs. If location is not specified, it will look for the keytab under the system's configuration directory.") + public String getKeytabLocation() { + return keytabLocation; + } + + public void setKeytabLocation(String keytabLocation) { + this.keytabLocation = keytabLocation; + } + + @Override public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + KerberosPrincipal kerberosPrincipal = (KerberosPrincipal) o; + return Objects.equals(this.principalName, kerberosPrincipal.principalName) + && Objects + .equals(this.keytabLocation, kerberosPrincipal.keytabLocation); + } + + @Override public int hashCode() { + return Objects.hash(principalName, keytabLocation); + } + + @Override public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class KerberosPrincipal {\n"); + + sb.append(" principalName: ").append(toIndentedString(principalName)) + .append("\n"); + sb.append(" keytabLocation: ").append(toIndentedString(keytabLocation)) + .append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index a3ba8c0..7ddaa01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -73,7 +73,6 @@ import org.apache.slider.common.Constants; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; @@ -686,12 +685,8 @@ private ApplicationId submitApp(Application app) // copy jars to hdfs and add to localResources addJarResource(appName, localResources); // add keytab if in secure env - addKeytabResourceIfSecure(sliderFileSystem, localResources, conf, appName); + addKeytabResourceIfSecure(sliderFileSystem, localResources, app); printLocalResources(localResources); - - //TODO SliderAMClientProvider#copyEnvVars - //TODO localResource putEnv - Map env = addAMEnv(conf); // create AM CLI @@ -902,38 +897,32 @@ private void persistApp(Path appDir, Application application) } private void addKeytabResourceIfSecure(SliderFileSystem fileSystem, - Map localResource, Configuration conf, - String appName) throws IOException, BadConfigException { + Map localResource, Application app) + throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return; } - String keytabPreInstalledOnHost = - conf.get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); - if (StringUtils.isEmpty(keytabPreInstalledOnHost)) { - String amKeytabName = - conf.get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - String keytabDir = conf.get(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); - Path keytabPath = - fileSystem.buildKeytabPath(keytabDir, amKeytabName, appName); - if (fileSystem.getFileSystem().exists(keytabPath)) { - LocalResource keytabRes = - fileSystem.createAmResource(keytabPath, LocalResourceType.FILE); - localResource - .put(SliderKeys.KEYTAB_DIR + "/" + amKeytabName, keytabRes); - log.info("Adding AM keytab on hdfs: " + keytabPath); - } else { - log.warn("No keytab file was found at {}.", keytabPath); - if (conf.getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) { - throw new BadConfigException("No keytab file was found at %s.", - keytabPath); - } else { - log.warn("The AM will be " - + "started without a kerberos authenticated identity. " - + "The application is therefore not guaranteed to remain " - + "operational beyond 24 hours."); - } - } + if (app.getKerberosPrincipal() == null || app.getKerberosPrincipal() + .getKeytabLocation().isEmpty()) { + log.info("No keytab will be localized for application " + app.getName()); + return; + } + + Path keytabOnhdfs = + new Path(app.getKerberosPrincipal().getKeytabLocation()); + String principalName = app.getKerberosPrincipal().getPrincipalName(); + if (!fileSystem.getFileSystem().exists(keytabOnhdfs)) { + log.warn(app.getName() + "'s keytab (principalName = " + principalName + + ") is not pre-installed on hdfs: " + keytabOnhdfs); + return; } + LocalResource keytabRes = + fileSystem.createAmResource(keytabOnhdfs, LocalResourceType.FILE); + localResource.put(String.format(SliderKeys.KEYTAB_LOCATION, app.getName()), + keytabRes); + log.info( + "Adding " + app.getName() + "'s keytab for localization from hdfs: " + + keytabOnhdfs); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 865562e..7b8c894 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -97,6 +97,7 @@ String HISTORY_FILENAME_SUFFIX = "json"; String HISTORY_FILENAME_PREFIX = "rolehistory-"; String KEYTAB_DIR = "keytabs"; + String KEYTAB_LOCATION = KEYTAB_DIR + "/%s" + ".keytab"; String RESOURCE_DIR = "resources"; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index e881edf..f220bb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -22,24 +22,6 @@ * These are the keys that can be added to conf/slider-client.xml. */ public interface SliderXmlConfKeys { - String PREFIX_PROVIDER = "slider.provider"; - /** - * pattern to identify a provider - * {@value} - */ - String KEY_PROVIDER = PREFIX_PROVIDER + ".%s"; - - /** - * conf option set to point to where the config came from - * {@value} - */ - String KEY_TEMPLATE_ORIGIN = "slider.template.origin"; - - /** - * Original name for the default FS. This is still - * expected by applications deployed - */ - String FS_DEFAULT_NAME_CLASSIC = "fs.default.name"; /** * Slider principal @@ -62,13 +44,6 @@ * queue name, by default let YARN pick the queue */ String KEY_YARN_QUEUE = "slider.yarn.queue"; - String DEFAULT_YARN_QUEUE = null; - - /** - * default priority - */ - String KEY_YARN_QUEUE_PRIORITY = "slider.yarn.queue.priority"; - int DEFAULT_YARN_QUEUE_PRIORITY = 1; String KEY_AM_RESOURCE_MEM = "slider.am.resource.memory"; @@ -92,29 +67,15 @@ */ String DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS = "750"; - /** - * - * Option for the permissions for the data directory itself: {@value} - */ - String DATA_DIRECTORY_PERMISSIONS = "slider.data.directory.permissions"; - - /** - * Default value for the data directory permissions: {@value} - */ - String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750"; String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH = "ipc.client.fallback-to-simple-auth-allowed"; String HADOOP_HTTP_FILTER_INITIALIZERS = "hadoop.http.filter.initializers"; - String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location"; String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name"; - /** Declare that a keytab must be provided */ - String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required"; String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir"; String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path"; String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name"; - String KEY_SECURITY_ENABLED = "site.global.security_enabled"; /** * Set to disable server-side checks for python, openssl &c. @@ -124,11 +85,6 @@ "slider.am.dependency.checks.disabled"; /** - * The path to the python executable utilized to launch the agent. - */ - String PYTHON_EXECUTABLE_PATH = "agent.python.exec.path"; - - /** * Flag to enable the insecure AM filter: {@value} */ String X_DEV_INSECURE_WS = "slider.feature.ws.insecure"; @@ -138,7 +94,6 @@ */ boolean X_DEV_INSECURE_DEFAULT = false; - /** * Flag to indicate the insecure AM filter is required for * complex REST Verbs: {@value}. @@ -147,16 +102,6 @@ */ boolean X_DEV_INSECURE_REQUIRED = true; - /** - * - */ - String KEY_IPC_CLIENT_RETRY_POLICY_ENABLED = - "slider.ipc.client.retry.enabled"; - boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true; - String KEY_IPC_CLIENT_RETRY_POLICY_SPEC = - "slider.ipc.client.retry.policy.spec"; - String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT = - "10000,6,60000,10"; //t1,n1,t2,n2,... String KEY_AM_LAUNCH_ENV = "slider.am.launch.env"; @@ -164,26 +109,7 @@ * From {@code DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY} */ String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal"; - String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal"; - - //Delegation token related keys - String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY - = "dfs.namenode.delegation.key.update-interval"; - long DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 * - 1000; // 1 day - String DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY - = "dfs.namenode.delegation.token.renew-interval"; - long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 * - 1000; // 1 day - String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY - = "dfs.namenode.delegation.token.max-lifetime"; - long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 * - 1000; // 7 days - String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY - = "dfs.namenode.delegation.token.always-use"; // for tests - boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false; - String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file"; String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved"; String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java index 25debdc..f301698 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java @@ -25,28 +25,14 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.core.exceptions.BadConfigException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.io.StringWriter; -import java.net.MalformedURLException; import java.net.URL; -import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -62,19 +48,6 @@ private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); /** - * Dump the (sorted) configuration - * @param conf config - * @return the sorted keyset - */ - public static Set dumpConf(Configuration conf) { - Set keys = sortedConfigKeys(conf); - for (String key : keys) { - log.info("{}={}", key, conf.get(key)); - } - return keys; - } - - /** * Take a configuration and return a sorted set * @param conf config * @return the sorted keyset @@ -123,25 +96,6 @@ public static void addConfigMap(Configuration config, /** - * Save a config file in a destination directory on a given filesystem - * @param systemConf system conf used for creating filesystems - * @param confToSave config to save - * @param confdir the directory path where the file is to go - * @param filename the filename - * @return the destination path where the file was saved - * @throws IOException IO problems - */ - public static Path saveConfig(Configuration systemConf, - Configuration confToSave, - Path confdir, - String filename) throws IOException { - FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); - Path destPath = new Path(confdir, filename); - saveConfig(fs, destPath, confToSave); - return destPath; - } - - /** * Save a config * @param fs filesystem * @param destPath dest to save @@ -171,45 +125,8 @@ public static String toXml(Configuration conf) throws IOException { conf.writeXml(writer); return writer.toString(); } - - /** - * This will load and parse a configuration to an XML document - * @param fs filesystem - * @param path path - * @return an XML document - * @throws IOException IO failure - */ - public Document parseConfiguration(FileSystem fs, - Path path) throws - IOException { - byte[] data = loadBytes(fs, path); - //this is here to track down a parse issue - //related to configurations - String s = new String(data, 0, data.length, "UTF-8"); - log.debug("XML resource {} is \"{}\"", path, s); -/* JDK7 - try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { - Document document = parseConfigXML(in); - return document; - } catch (ParserConfigurationException | SAXException e) { - throw new IOException(e); - } -*/ - ByteArrayInputStream in= null; - try { - in = new ByteArrayInputStream(data); - Document document = parseConfigXML(in); - return document; - } catch (ParserConfigurationException e) { - throw new IOException(e); - } catch (SAXException e) { - throw new IOException(e); - } finally { - IOUtils.closeStream(in); - } - } public static byte[] loadBytes(FileSystem fs, Path path) throws IOException { int len = (int) fs.getLength(path); @@ -259,185 +176,6 @@ public static Configuration loadConfiguration(FileSystem fs, /** - * Generate a config file in a destination directory on the local filesystem - * @param confdir the directory path where the file is to go - * @param filename the filename - * @return the destination path - */ - public static File saveConfig(Configuration generatingConf, - File confdir, - String filename) throws IOException { - - - File destPath = new File(confdir, filename); - OutputStream fos = new FileOutputStream(destPath); - try { - generatingConf.writeXml(fos); - } finally { - IOUtils.closeStream(fos); - } - return destPath; - } - - /** - * Parse an XML Hadoop configuration into an XML document. x-include - * is supported, but as the location isn't passed in, relative - * URIs are out. - * @param in instream - * @return a document - * @throws ParserConfigurationException parser feature problems - * @throws IOException IO problems - * @throws SAXException XML is invalid - */ - public static Document parseConfigXML(InputStream in) throws - ParserConfigurationException, - IOException, - SAXException { - DocumentBuilderFactory docBuilderFactory - = DocumentBuilderFactory.newInstance(); - //ignore all comments inside the xml file - docBuilderFactory.setIgnoringComments(true); - - //allow includes in the xml file - docBuilderFactory.setNamespaceAware(true); - docBuilderFactory.setXIncludeAware(true); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - return builder.parse(in); - } - - /** - * Load a Hadoop configuration from a local file. - * @param file file to load - * @return a configuration which hasn't actually had the load triggered - * yet. - * @throws FileNotFoundException file is not there - * @throws IOException any other IO problem - */ - public static Configuration loadConfFromFile(File file) throws - IOException { - return loadConfFromFile(file, false); - } - - /** - * - * Load a Hadoop configuration from a local file. - * @param file file to load - * @param loadDefaults flag to indicate if the defaults should be loaded yet - * @return a configuration which hasn't actually had the load triggered - * yet. - * @throws FileNotFoundException file is not there - * @throws IOException any other IO problem - */ - public static Configuration loadConfFromFile(File file, - boolean loadDefaults) throws IOException { - if (!file.exists()) { - throw new FileNotFoundException("File not found :" - + file.getAbsoluteFile()); - } - Configuration conf = new Configuration(loadDefaults); - try { - conf.addResource(file.toURI().toURL()); - } catch (MalformedURLException e) { - // should never happen... - throw new IOException( - "File " + file.toURI() + " doesn't have a valid URL"); - } - return conf; - } - - /** - * Add a configuration from a file to an existing configuration - * @param conf existing configuration - * @param file file to load - * @param overwrite flag to indicate new values should overwrite the predecessor - * @return the merged configuration - * @throws IOException - */ - public static Configuration addConfigurationFile(Configuration conf, - File file, boolean overwrite) - throws IOException { - Configuration c2 = loadConfFromFile(file, false); - mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite); - return conf; - } - - /** - * Add the system env variables with the given prefix (by convention, env.) - * @param conf existing configuration - * @param prefix prefix - */ - public static void addEnvironmentVariables(Configuration conf, String prefix) { - Map env = System.getenv(); - for (Map.Entry entry : env.entrySet()) { - conf.set(prefix + entry.getKey(),entry.getValue(), "env"); - } - } - - /** - * looks for the config under $confdir/$templateFilename; if not there - * loads it from /conf/templateFile. - * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the - * origin to help debug what's happening - * @param systemConf system conf - * @param confdir conf dir in FS - * @param templateFilename filename in the confdir - * @param fallbackResource resource to fall back on - * @return loaded conf - * @throws IOException IO problems - */ - public static Configuration loadTemplateConfiguration(Configuration systemConf, - Path confdir, - String templateFilename, - String fallbackResource) throws - IOException { - FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); - - Path templatePath = new Path(confdir, templateFilename); - return loadTemplateConfiguration(fs, templatePath, fallbackResource); - } - - /** - * looks for the config under $confdir/$templateFilename; if not there - * loads it from /conf/templateFile. - * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the - * origin to help debug what's happening. - * @param fs Filesystem - * @param templatePath HDFS path for template - * @param fallbackResource resource to fall back on, or "" for no fallback - * @return loaded conf - * @throws IOException IO problems - * @throws FileNotFoundException if the path doesn't have a file and there - * was no fallback. - */ - public static Configuration loadTemplateConfiguration(FileSystem fs, - Path templatePath, - String fallbackResource) - throws IOException { - Configuration conf; - String origin; - if (fs.exists(templatePath)) { - log.debug("Loading template configuration {}", templatePath); - conf = loadConfiguration(fs, templatePath); - origin = templatePath.toString(); - } else { - if (fallbackResource.isEmpty()) { - throw new FileNotFoundException("No config file found at " + templatePath); - } - log.debug("Template {} not found" + - " -reverting to classpath resource {}", templatePath, fallbackResource); - conf = new Configuration(false); - conf.addResource(fallbackResource); - origin = "Resource " + fallbackResource; - } - //force a get - conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN); - //now set the origin - conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin); - return conf; - } - - - /** * For testing: dump a configuration * @param conf configuration * @return listing in key=value style @@ -525,61 +263,6 @@ public static URL getResourceUrl(String resource) { } /** - * Load a resource that must be on the classpath - * @param resource the resource name - * @return the loaded configuration - * @throws FileNotFoundException if the resource is missing - */ - public static Configuration loadMandatoryResource(String resource) - throws FileNotFoundException { - Configuration conf = new Configuration(false); - URL resURL = getResourceUrl(resource); - if (resURL != null) { - log.debug("loaded resources from {}", resURL); - conf.addResource(resource); - } else { - throw new FileNotFoundException(resource); - } - return conf; - } - - /** - * Propagate a property from a source to a dest config, with a best-effort - * attempt at propagating the origin. - * If the - * @param dest destination - * @param src source - * @param key key to try to copy - * @return true if the key was found and propagated - */ - public static boolean propagate(Configuration dest, - Configuration src, - String key) { - String val = src.get(key); - if (val != null) { - String[] origin = src.getPropertySources(key); - if (origin != null && origin.length > 0) { - dest.set(key, val, origin[0]); - } else { - dest.set(key, val); - return true; - } - } - return false; - } - - - /** - * Take a configuration, return a hash map - * @param conf conf - * @return hash map - */ - public static Map buildMapFromConfiguration(Configuration conf) { - Map map = new HashMap(); - return SliderUtils.mergeEntries(map, conf); - } - - /** * This goes through the keyset of one configuration and retrieves each value * from a value source -a different or the same configuration. This triggers * the property resolution process of the value, resolving any variables against diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index d58ecaa..7ad78a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -183,77 +183,6 @@ public void substituteMapWithTokens(Map configs, } } - /** - * Get resource requirements from a String value. If value isn't specified, - * use the default value. If value is greater than max, use the max value. - * @param val string value - * @param defVal default value - * @param maxVal maximum value - * @return int resource requirement - */ - public int getRoleResourceRequirement(String val, - int defVal, - int maxVal) { - if (val==null) { - val = Integer.toString(defVal); - } - Integer intVal; - if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) { - intVal = maxVal; - } else { - intVal = Integer.decode(val); - } - return intVal; - } - - - /** - * Localize the service keytabs for the application. - * @param launcher container launcher - * @param fileSystem file system - * @throws IOException trouble uploading to HDFS - */ - public void localizeServiceKeytabs(ContainerLauncher launcher, - SliderFileSystem fileSystem, Application application) throws IOException { - - Configuration conf = application.getConfiguration(); - String keytabPathOnHost = - conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); - if (SliderUtils.isUnset(keytabPathOnHost)) { - String amKeytabName = - conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - String keytabDir = - conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); - // we need to localize the keytab files in the directory - Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null, - application.getName()); - boolean serviceKeytabsDeployed = false; - if (fileSystem.getFileSystem().exists(keytabDirPath)) { - FileStatus[] keytabs = fileSystem.getFileSystem().listStatus( - keytabDirPath); - LocalResource keytabRes; - for (FileStatus keytab : keytabs) { - if (!amKeytabName.equals(keytab.getPath().getName()) - && keytab.getPath().getName().endsWith(".keytab")) { - serviceKeytabsDeployed = true; - log.info("Localizing keytab {}", keytab.getPath().getName()); - keytabRes = fileSystem.createAmResource(keytab.getPath(), - LocalResourceType.FILE); - launcher.addLocalResource(KEYTAB_DIR + "/" + - keytab.getPath().getName(), - keytabRes); - } - } - } - if (!serviceKeytabsDeployed) { - log.warn("No service keytabs for the application have been localized. " - + "If the application requires keytabs for secure operation, " - + "please ensure that the required keytabs have been uploaded " - + "to the folder {}", keytabDirPath); - } - } - } - public static void addEnvForSubstitution(Map env, Map tokensForSubstitution) { if (env == null || env.isEmpty() || tokensForSubstitution == null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 0c3fcea..042f34c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -23,6 +23,7 @@ import com.google.common.base.Preconditions; import com.google.protobuf.BlockingService; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; @@ -89,6 +90,7 @@ import org.apache.slider.api.resource.Component; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.SliderAMArgs; import org.apache.slider.common.params.SliderAMCreateAction; @@ -386,7 +388,6 @@ * Is security enabled? * Set early on in the {@link #createAndRunCluster(String)} operation. */ - private boolean securityEnabled; private ContentCache contentCache; /** @@ -484,7 +485,6 @@ public synchronized void serviceInit(Configuration conf) throws Exception { timelineServiceEnabled = true; log.info("Enabled YARN timeline service v2. "); } - //init all child services super.serviceInit(conf); } @@ -676,7 +676,7 @@ private int createAndRunCluster(String appName) throws Throwable { // set up secret manager secretManager = new ClientToAMTokenSecretManager(appAttemptID, null); - if (securityEnabled) { + if (UserGroupInformation.isSecurityEnabled()) { // fix up the ACLs if they are not set String acls = serviceConf.get(KEY_PROTOCOL_ACL); if (acls == null) { @@ -741,29 +741,9 @@ private int createAndRunCluster(String appName) throws Throwable { stripAMRMToken(); -// if (securityEnabled) { -// secretManager.setMasterKey( -// amRegistrationData.getClientToAMTokenMasterKey().array()); -// applicationACLs = amRegistrationData.getApplicationACLs(); -// -// //tell the server what the ACLs are -// rpcService.getServer().refreshServiceAcl(serviceConf, -// new SliderAMPolicyProvider()); -// if (securityConfiguration.isKeytabProvided()) { -// // perform keytab based login to establish kerberos authenticated -// // principal. Can do so now since AM registration with RM above required -// // tokens associated to principal -// String principal = securityConfiguration.getPrincipal(); -// //TODO read key tab file from slider-am.xml -// File localKeytabFile = new File("todo"); -//// securityConfiguration.getKeytabFile(new AggregateConf()); -// // Now log in... -// login(principal, localKeytabFile); -// // obtain new FS reference that should be kerberos based and different -// // than the previously cached reference -// fs = new SliderFileSystem(serviceConf); -// } -// } + if (UserGroupInformation.isSecurityEnabled()) { + fs = doSecureLogin(appName, fs, serviceConf); + } // YARN client. // Important: this is only valid at startup, and must be executed within @@ -898,6 +878,41 @@ private int createAndRunCluster(String appName) throws Throwable { return finish(); } + private SliderFileSystem doSecureLogin(String appName, SliderFileSystem fs, + Configuration serviceConf) throws IOException, SliderException { + secretManager + .setMasterKey(amRegistrationData.getClientToAMTokenMasterKey().array()); + String principal = application.getKerberosPrincipal().getPrincipalName(); + File keytab = new File( + String.format(SliderKeys.KEYTAB_LOCATION, application.getName())); + if (!keytab.exists()) { + String keytabPreInstalled = + getConfig().get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); + if (!StringUtils.isEmpty(keytabPreInstalled)) { + keytab = new File(keytabPreInstalled); + log.info("Using pre-installed keytab at: " + keytabPreInstalled); + } + } + if (!keytab.exists()) { + log.warn("No keytab exists for " + appName + ": " + keytab); + return fs; + } + if (StringUtils.isEmpty(principal)) { + // get Principal from the local AM config + principal = getConfig().get(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL); + if (StringUtils.isEmpty(principal)) { + principal = UserGroupInformation.getLoginUser().getShortUserName(); + log.info("No principal set in the slider configuration. Will use AM " + + "login identity {} to attempt keytab-based login", principal); + } + } + login(principal, keytab); + // obtain new FS reference that should be kerberos based and different + // than the previously cached reference + return new SliderFileSystem(serviceConf); + + } + /** * Get the YARN application Attempt report as the logged in user * @param yarnClient client to the RM @@ -911,7 +926,7 @@ private ApplicationAttemptReport getApplicationAttemptReport( throws YarnException, IOException, InterruptedException { Preconditions.checkNotNull(yarnClient, "Null Yarn client"); ApplicationAttemptReport report; - if (securityEnabled) { + if (UserGroupInformation.isSecurityEnabled()) { UserGroupInformation ugi = UserGroupInformation.getLoginUser(); report = ugi.doAs(new PrivilegedExceptionAction() { @Override @@ -937,7 +952,7 @@ public ApplicationAttemptReport run() throws Exception { throws IOException, YarnException, InterruptedException { Preconditions.checkNotNull(yarnClient, "Null Yarn client"); List nodeReports; - if (securityEnabled) { + if (UserGroupInformation.isSecurityEnabled()) { nodeReports = UserGroupInformation.getLoginUser().doAs( new PrivilegedExceptionAction>() { @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java deleted file mode 100644 index cc19eee..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.security; - -import com.google.common.base.Preconditions; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; -import static org.apache.slider.core.main.LauncherExitCodes.EXIT_UNAUTHORIZED; - -import org.apache.slider.api.resource.Application; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.SliderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; - -/** - * Class keeping code security information - */ -public class SecurityConfiguration { - - protected static final Logger log = - LoggerFactory.getLogger(SecurityConfiguration.class); - private final Configuration configuration; - private final Application application; - private String clusterName; - - public SecurityConfiguration(Configuration configuration, - Application application, - String clusterName) throws SliderException { - Preconditions.checkNotNull(configuration); - Preconditions.checkNotNull(application); - Preconditions.checkNotNull(clusterName); - this.configuration = configuration; - this.application = application; - this.clusterName = clusterName; - validate(); - } - - private void validate() throws SliderException { - if (isSecurityEnabled()) { - // TODO use AM configuration rather than app configuration - String principal = application.getConfiguration().getProperty( - SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL); - if(SliderUtils.isUnset(principal)) { - // if no login identity is available, fail - UserGroupInformation loginUser = null; - try { - loginUser = getLoginUser(); - } catch (IOException e) { - throw new SliderException(EXIT_UNAUTHORIZED, e, - "No principal configured for the application and " - + "exception raised during retrieval of login user. " - + "Unable to proceed with application " - + "initialization. Please ensure a value " - + "for %s exists in the application " - + "configuration or the login issue is addressed", - SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL); - } - if (loginUser == null) { - throw new SliderException(EXIT_UNAUTHORIZED, - "No principal configured for the application " - + "and no login user found. " - + "Unable to proceed with application " - + "initialization. Please ensure a value " - + "for %s exists in the application " - + "configuration or the login issue is addressed", - SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL); - } - } - // ensure that either local or distributed keytab mechanism is enabled, - // but not both - String keytabFullPath = application.getConfiguration().getProperty( - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); - String keytabName = application.getConfiguration().getProperty( - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) { - throw new SliderException(EXIT_UNAUTHORIZED, - "Both a keytab on the cluster host (%s) and a" - + " keytab to be retrieved from HDFS (%s) are" - + " specified. Please configure only one keytab" - + " retrieval mechanism.", - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - - } - } - } - - protected UserGroupInformation getLoginUser() throws IOException { - return UserGroupInformation.getLoginUser(); - } - - public boolean isSecurityEnabled() { - return SliderUtils.isHadoopClusterSecure(configuration); - } - - public String getPrincipal() throws IOException { - String principal = application.getConfiguration().getProperty( - SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL); - if (SliderUtils.isUnset(principal)) { - principal = UserGroupInformation.getLoginUser().getShortUserName(); - log.info("No principal set in the slider configuration. Will use AM " + - "login identity {} to attempt keytab-based login", principal); - } - - return principal; - } - - public boolean isKeytabProvided() { - String keytabLocalPath = application.getConfiguration().getProperty( - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); - String keytabName = application.getConfiguration().getProperty( - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - return StringUtils.isNotBlank(keytabLocalPath) - || StringUtils.isNotBlank(keytabName); - - } - - public File getKeytabFile() - throws SliderException, IOException { - //TODO implement this for dash semantic - String keytabFullPath = application.getConfiguration().getProperty( - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); - File localKeytabFile; - if (SliderUtils.isUnset(keytabFullPath)) { - // get the keytab - String keytabName = application.getConfiguration().getProperty( - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - log.info("No host keytab file path specified. Will attempt to retrieve" - + " keytab file {} as a local resource for the container", - keytabName); - // download keytab to local, protected directory - localKeytabFile = new File(SliderKeys.KEYTAB_DIR, keytabName); - } else { - log.info("Using host keytab file {} for login", keytabFullPath); - localKeytabFile = new File(keytabFullPath); - } - return localKeytabFile; - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java deleted file mode 100644 index 5a19a3a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/security/TestSecurityConfiguration.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.security; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.slider.api.resource.Application; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.core.exceptions.SliderException; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test security configuration. - */ -public class TestSecurityConfiguration { - - @Test - public void testValidLocalConfiguration() throws Throwable { - Configuration config = new Configuration(); - config.set(CommonConfigurationKeysPublic - .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - Map compOps = new HashMap<>(); - compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test"); - compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, - "/some/local/path"); - Application application = new Application().configuration(new org.apache - .slider.api.resource.Configuration().properties(compOps)); - - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, application, "testCluster"); - } - - @Test - public void testValidDistributedConfiguration() throws Throwable { - Configuration config = new Configuration(); - config.set(CommonConfigurationKeysPublic - .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - Map compOps = new HashMap<>(); - compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test"); - compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); - Application application = new Application().configuration(new org.apache - .slider.api.resource.Configuration().properties(compOps)); - - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, application, "testCluster"); - } - - @Test - public void testMissingPrincipalNoLoginWithDistributedConfig() throws - Throwable { - Configuration config = new Configuration(); - config.set(CommonConfigurationKeysPublic - .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - Map compOps = new HashMap<>(); - compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); - Application application = new Application().configuration(new org.apache - .slider.api.resource.Configuration().properties(compOps)); - - try { - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, application, "testCluster") { - @Override - protected UserGroupInformation getLoginUser() throws - IOException { - return null; - } - }; - fail("expected SliderException"); - } catch (SliderException e) { - // expected - } - } - - @Test - public void testMissingPrincipalNoLoginWithLocalConfig() throws Throwable { - Configuration config = new Configuration(); - config.set(CommonConfigurationKeysPublic - .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - Map compOps = new HashMap<>(); - compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, - "/some/local/path"); - Application application = new Application().configuration(new org.apache - .slider.api.resource.Configuration().properties(compOps)); - - try { - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, application, "testCluster") { - @Override - protected UserGroupInformation getLoginUser() throws IOException { - return null; - } - }; - fail("expected SliderException"); - } catch (SliderException e) { - // expected - } - } - - @Test - public void testBothKeytabMechanismsConfigured() throws Throwable { - Configuration config = new Configuration(); - config.set(CommonConfigurationKeysPublic - .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - Map compOps = new HashMap<>(); - compOps.put(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL, "test"); - compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, - "/some/local/path"); - compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); - Application application = new Application().configuration(new org.apache - .slider.api.resource.Configuration().properties(compOps)); - - try { - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, application, - "testCluster"); - fail("expected SliderException"); - } catch (SliderException e) { - // expected - } - } - - @Test - public void testMissingPrincipalButLoginWithDistributedConfig() throws - Throwable { - Configuration config = new Configuration(); - config.set(CommonConfigurationKeysPublic - .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - Map compOps = new HashMap<>(); - compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); - Application application = new Application().configuration(new org.apache - .slider.api.resource.Configuration().properties(compOps)); - - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, application, "testCluster"); - } - - @Test - public void testMissingPrincipalButLoginWithLocalConfig() throws Throwable { - Configuration config = new Configuration(); - config.set(CommonConfigurationKeysPublic - .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - Map compOps = new HashMap<>(); - compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, - "/some/local/path"); - Application application = new Application().configuration(new org.apache - .slider.api.resource.Configuration().properties(compOps)); - - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, application, "testCluster"); - } - - @Test - public void testKeypathLocationOnceLocalized() throws Throwable { - Configuration config = new Configuration(); - config.set(CommonConfigurationKeysPublic - .HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - Map compOps = new HashMap<>(); - compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); - Application application = new Application().configuration(new org.apache - .slider.api.resource.Configuration().properties(compOps)); - - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, application, "testCluster"); - - assertEquals(new File(SliderKeys.KEYTAB_DIR, "some.keytab") - .getAbsolutePath(), - securityConfiguration.getKeytabFile().getAbsolutePath()); - } - - @Test - public void testAMKeytabProvided() throws Throwable { - Configuration config = new Configuration(); - Map compOps = new HashMap<>(); - compOps.put(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH, " "); - Application application = new Application().configuration(new org.apache - .slider.api.resource.Configuration().properties(compOps)); - - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, application, "testCluster"); - assertFalse(securityConfiguration.isKeytabProvided()); - - compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, ""); - assertFalse(securityConfiguration.isKeytabProvided()); - - compOps.put(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME, "some.keytab"); - assertTrue(securityConfiguration.isKeytabProvided()); - } - -}