diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 4d74f7622f89e9edd9a9cff3b5920e51ab7d202c..f822e57c354ca6972fdc3291f816ed532d5cbc21 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -24,7 +24,8 @@ - + + Factory to create client IPC classes. yarn.ipc.client.factory.class @@ -46,7 +47,8 @@ org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC - + + The hostname of the RM. yarn.resourcemanager.hostname @@ -131,6 +133,32 @@ + + The Kerberos keytab file to be used for spnego filter for the RM web + interface. + + yarn.resourcemanager.webapp.spnego-keytab-file + + + + + + The Kerberos principal to be used for spnego filter for the RM web + interface. + + yarn.resourcemanager.webapp.spnego-principal + + + + + + Add button to kill application in the RM Application view. + + yarn.resourcemanager.webapp.ui-actions.enabled + true + + + yarn.resourcemanager.resource-tracker.address ${yarn.resourcemanager.hostname}:8031 @@ -268,7 +296,16 @@ - Enable RM to recover state after starting. If true, then + + Used by node labels. If set to true, the port should be included in the + node name. Only usable if your scheduler supports node labels. + + yarn.scheduler.include-port-in-node-name + false + + + + Enable RM to recover state after starting. If true, then yarn.resourcemanager.store.class must be specified. yarn.resourcemanager.recovery.enabled false @@ -661,6 +698,16 @@ + + The value specifies the file system (e.g. HDFS) path where ResourceManager + loads configuration if yarn.resourcemanager.configuration.provider-class + is set to org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider. + + yarn.resourcemanager.configuration.file-system-based-store + /yarn/conf + + + The setting that controls whether yarn system metrics is published on the timeline server or not by RM. yarn.resourcemanager.system-metrics-publisher.enabled @@ -674,7 +721,118 @@ 10 - + + + RM DelegationTokenRenewer thread count + + yarn.resourcemanager.delegation-token-renewer.thread-count + 50 + + + + + RM secret key update interval in ms + + yarn.resourcemanager.delegation.key.update-interval + 86400000 + + + + + RM delegation token maximum lifetime in ms + + yarn.resourcemanager.delegation.token.max-lifetime + 604800000 + + + + + RM delegation token update interval in ms + + yarn.resourcemanager.delegation.token.renew-interval + 86400000 + + + + + Thread pool size for RMApplicationHistoryWriter. + + yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size + 10 + + + + + Comma-separated list of values (in minutes) for schedule queue related + metrics. + + yarn.resourcemanager.metrics.runtime.buckets + 60,300,1440 + + + + + Interval for the roll over for the master key used to generate + NodeManager tokens. It is expected to be set to a value much larger + than yarn.nm.liveness-monitor.expiry-interval-ms. + + yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs + 86400 + + + + + Flag to enable the ResourceManager reservation system. + + yarn.resourcemanager.reservation-system.enable + false + + + + + The Java class to use as the ResourceManager reservation system. + By default, is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityReservationSystem + when using CapacityScheduler and is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.FairReservationSystem + when using FairScheduler. + + yarn.resourcemanager.reservation-system.class + + + + + + The plan follower policy class name to use for the ResourceManager + reservation system. + By default, is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacitySchedulerPlanFollower + is used when using CapacityScheduler, and is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.FairSchedulerPlanFollower + when using FairScheduler. + + yarn.resourcemanager.reservation-system.plan.follower + + + + + + Step size of the reservation system in ms + + yarn.resourcemanager.reservation-system.planfollower.time-step + 1000 + + + + + The expiry interval for a container + + yarn.resourcemanager.rm.container-allocation.expiry-interval-ms + 600000 + + + + The hostname of the NM. yarn.nodemanager.hostname @@ -820,6 +978,13 @@ + + yarn.nodemanager.container-localizer.java.opts + -Xmx256m + + + + Where to store container logs. An application's localized log directory will be found in ${yarn.nodemanager.log-dirs}/application_${appid}. Individual containers' log directories will be below this, in directories @@ -860,6 +1025,17 @@ + + How long for ResourceManager to wait for NodeManager to report its + log aggregation status. If waiting time of which the log aggregation + status is reported from NodeManager exceeds the configured value, RM + will report log aggregation status for this NodeManager as TIME_OUT + + yarn.log-aggregation-status.time-out.ms + 60000 + + + Time in seconds to retain user logs. Only applicable if log aggregation is disabled @@ -937,6 +1113,32 @@ + + The https adddress of the NM web application. + + yarn.nodemanager.webapp.https.address + + + + + + The Kerberos keytab file to be used for spnego filter for the NM web + interface. + + yarn.nodemanager.webapp.spnego-keytab-file + + + + + + The Kerberos principal to be used for spnego filter for the NM web + interface. + + yarn.nodemanager.webapp.spnego-principal + + + + How often to monitor containers. yarn.nodemanager.container-monitor.interval-ms 3000 @@ -1044,6 +1246,12 @@ + Delay in ms between attempts to remove linux cgroup + yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms + 20 + + + This determines which of the two modes that LCE should use on a non-secure cluster. If this value is set to true, then all containers will be launched as the user specified in @@ -1097,6 +1305,23 @@ + + Interval of time the linux container executor should try cleaning up + cgroups entry when cleaning up a container. + + yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms + 1000 + + + + + The UNIX group that the linux-container-executor should run as. + + yarn.nodemanager.linux-container-executor.group + + + + T-file compression types used to compress aggregated logs. yarn.nodemanager.log-aggregation.compression-type none @@ -1204,6 +1429,66 @@ + + Adjustment to the container OS scheduling priority. In Linux, passed + directly to the nice command. + + yarn.nodemanager.container-executor.os.sched.priority.adjustment + 0 + + + + + Flag to enable container metrics + + yarn.nodemanager.container-metrics.enable + true + + + + + Container metrics flush period in ms. Set to -1 for flush on completion. + + yarn.nodemanager.container-metrics.period-ms + -1 + + + + + Class used to calculate current container resource utilization. + + yarn.nodemanager.container-monitor.process-tree.class + + + + + + Flag to enable NodeManager disk health checker + + yarn.nodemanager.disk-health-checker.enable + true + + + + + Number of threads to use in NM log cleanup. Used when log aggregation + is disabled. + + yarn.nodemanager.log.deletion-threads-count + 4 + + + + + The Windows group that the windows-container-executor should run as. + + yarn.nodemanager.windows-secure-container-executor.group + + + + + + yarn.nodemanager.docker-container-executor.exec-name /usr/bin/docker @@ -1211,24 +1496,28 @@ - - yarn.nodemanager.aux-services.mapreduce_shuffle.class - org.apache.hadoop.mapred.ShuffleHandler + + The Docker image name to use for DockerContainerExecutor + + yarn.nodemanager.docker-container-executor.image-name + - - mapreduce.job.jar - - + mapreduce.job.hdfs-servers ${fs.defaultFS} - - + + yarn.nodemanager.aux-services.mapreduce_shuffle.class + org.apache.hadoop.mapred.ShuffleHandler + + + + The kerberos principal for the proxy, if the proxy is not running as part of the RM. @@ -1249,8 +1538,8 @@ - - + + CLASSPATH for YARN applications. A comma-separated list @@ -1277,7 +1566,7 @@ - + Indicate to clients whether timeline service is enabled or not. @@ -1452,6 +1741,7 @@ + Whether the shared cache is enabled yarn.sharedcache.enabled @@ -1593,7 +1883,93 @@ 20 - + + + ACL protocol for use in the Timeline server. + + security.applicationhistory.protocol.acl + + + + + + + + Set to true for MiniYARNCluster unit tests + + yarn.is.minicluster + false + + + + + Set for MiniYARNCluster unit tests to control resource monitoring + + yarn.minicluster.control-resource-monitoring + false + + + + + Set to false in order to allow MiniYARNCluster to run tests without + port conflicts. + + yarn.minicluster.fixed.ports + false + + + + + Set to false in order to allow the NodeManager in MiniYARNCluster to + use RPC to talk to the RM. + + yarn.minicluster.use-rpc + false + + + + + As yarn.nodemanager.resource.memory-mb property but for the NodeManager + in a MiniYARNCluster. + + yarn.minicluster.yarn.nodemanager.resource.memory-mb + + + + + + + + Enable node labels feature + + yarn.node-labels.enabled + false + + + + + + yarn.node-labels.fs-store.retry-policy-spec + + + + + + URI for NodeLabelManager + + yarn.node-labels.fs-store.root-dir + + + + + + + yarn.node-labels.configuration-type + centralized + + + + The interval that the yarn client library uses to poll the completion status of the asynchronous API of application client protocol. @@ -1603,7 +1979,25 @@ - RSS usage of a process computed via + + The duration (in ms) the YARN client waits for an expected state change + to occur. -1 means unlimited wait time. + + yarn.client.application-client-protocol.poll-timeout-ms + -1 + + + + + Deprecated in favor of + yarn.client.application-client-protocol.poll-timeout-ms. + + yarn.client.app-submission.poll-interval + + + + + RSS usage of a process computed via /proc/pid/stat is not very accurate as it includes shared pages of a process. /proc/pid/smaps provides useful information like Private_Dirty, Private_Clean, Shared_Dirty, Shared_Clean which can be used @@ -1616,6 +2010,157 @@ + + URL for log aggregation server + + yarn.log.server.url + + + + + + RM Application Tracking URL + + yarn.tracking.url.generator + + + + + + Class to be used for YarnAuthorizationProvider + + yarn.authorization-provider + + + + + + + + Is the registry enabled: does the RM start it up, + create the user and system paths, and purge + service records when containers, application attempts + and applications complete + + hadoop.registry.rm.enabled + false + + + + + Root path in the ZK tree for the registry + + hadoop.registry.zk.root + /registry + + + + + Zookeeper session timeout in milliseconds + + hadoop.registry.zk.session.timeout.ms + 60000 + + + + + Zookeeper session timeout in milliseconds + + hadoop.registry.zk.connection.timeout.ms + 15000 + + + + + Zookeeper connection retry count before failing + + hadoop.registry.zk.retry.times + 5 + + + + + Zookeeper connect interval in milliseconds + + hadoop.registry.zk.retry.interval.ms + 1000 + + + + + Zookeeper retry limit in milliseconds, during + exponential backoff: {@value} + + This places a limit even + if the retry times and interval limit, combined + with the backoff policy, result in a long retry + period + + hadoop.registry.zk.retry.ceiling.ms + 60000 + + + + + List of hostname:port pairs defining the + zookeeper quorum binding for the registry + + hadoop.registry.zk.quorum + localhost:2181 + + + + + Key to set if the registry is secure. Turning it on + changes the permissions policy from "open access" + to restrictions on kerberos with the option of + a user adding one or more auth key pairs down their + own tree. + + hadoop.registry.secure + false + + + + + A comma separated list of Zookeeper ACL identifiers with + system access to the registry in a secure cluster. + + These are given full access to all entries. + + If there is an "@" at the end of a SASL entry it + instructs the registry client to append the default kerberos domain. + + hadoop.registry.system.acls + sasl:yarn@, sasl:mapred@, sasl:mapred@hdfs@ + + + + + The kerberos realm: used to set the realm of + system principals which do not declare their realm, + and any other accounts that need the value. + + If empty, the default realm of the running process + is used. + + If neither are known and the realm is needed, then the registry + service/client will fail. + + hadoop.registry.kerberos.realm + + + + + + Key to define the JAAS context. Used in secure + mode + + hadoop.registry.jaas.context + Client + + + Defines how often NMs wake up to upload log files. The default value is -1. By default, the logs will be uploaded when the application is finished. By setting this configure, logs can be uploaded @@ -1625,4 +2170,5 @@ yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds -1 + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 9075d9f6dd3e589399b1c6d46ab40f77f181fa3b..ae8dfd202810674936d842d3a6f1b41786ccc2c0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -37,14 +37,14 @@ public void initializeMemberVariables() { xmlFilename = new String("yarn-default.xml"); configurationClasses = new Class[] { YarnConfiguration.class }; - // Allocate for usage configurationPropsToSkipCompare = new HashSet(); + configurationPrefixToSkipCompare = new HashSet(); // Set error modes errorIfMissingConfigProps = true; - errorIfMissingXmlProps = false; + errorIfMissingXmlProps = true; // Specific properties to skip configurationPropsToSkipCompare @@ -79,6 +79,33 @@ public void initializeMemberVariables() { configurationPropsToSkipCompare .add(YarnConfiguration .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SCM_STORE_CLASS); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL); + + // Ignore all YARN Application Timeline Service (version 1) properties + configurationPrefixToSkipCompare.add("yarn.timeline-service."); + + // Used as Java command line properties, not XML + configurationPrefixToSkipCompare.add("yarn.app.container"); + + // Ignore NodeManager "work in progress" variables + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_INTERFACE); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_DISK_RESOURCE_ENABLED); + + // Set by container-executor.cfg + configurationPrefixToSkipCompare.add(YarnConfiguration.NM_USER_HOME_DIR); // Allocate for usage xmlPropsToSkipCompare = new HashSet(); @@ -94,13 +121,11 @@ public void initializeMemberVariables() { // Used in the XML file as a variable reference internal to the XML file xmlPropsToSkipCompare.add("yarn.nodemanager.hostname"); - xmlPropsToSkipCompare.add("yarn.timeline-service.hostname"); - // Currently defined in TimelineAuthenticationFilterInitializer - xmlPrefixToSkipCompare.add("yarn.timeline-service.http-authentication"); + // Ignore all YARN Application Timeline Service (version 1) properties + xmlPrefixToSkipCompare.add("yarn.timeline-service"); // Currently defined in RegistryConstants xmlPrefixToSkipCompare.add("hadoop.registry"); } - }