diff --git hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
index f9647359d736bcb8a036820577a7a48426d7aab2..f8bc95b9a5479ca5d84890794df475985e5c168f 100644
--- hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
+++ hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
@@ -276,6 +276,7 @@ The following table lists the configuration property names that are deprecated i
| user.name | mapreduce.job.user.name |
| webinterface.private.actions | mapreduce.jobtracker.webinterface.trusted |
| yarn.app.mapreduce.yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts |
+| yarn.client.app-submission.poll-interval | yarn.client.application-client-protocol.poll-timeout-ms |
The following table lists additional changes to some configuration properties:
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1dd88bdcd06b25471c8f41b675580f8bc3026dbe..134b6d0c6a7f974801d89202d34c75dd698aeed7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -24,7 +24,8 @@
-
+
+
Factory to create client IPC classes.
yarn.ipc.client.factory.class
@@ -46,7 +47,8 @@
org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC
-
+
+
The hostname of the RM.
yarn.resourcemanager.hostname
@@ -131,6 +133,32 @@
+
+ The Kerberos keytab file to be used for spnego filter for the RM web
+ interface.
+
+ yarn.resourcemanager.webapp.spnego-keytab-file
+
+
+
+
+
+ The Kerberos principal to be used for spnego filter for the RM web
+ interface.
+
+ yarn.resourcemanager.webapp.spnego-principal
+
+
+
+
+
+ Add button to kill application in the RM Application view.
+
+ yarn.resourcemanager.webapp.ui-actions.enabled
+ true
+
+
+
yarn.resourcemanager.resource-tracker.address
${yarn.resourcemanager.hostname}:8031
@@ -268,7 +296,16 @@
- Enable RM to recover state after starting. If true, then
+
+ Used by node labels. If set to true, the port should be included in the
+ node name. Only usable if your scheduler supports node labels.
+
+ yarn.scheduler.include-port-in-node-name
+ false
+
+
+
+ Enable RM to recover state after starting. If true, then
yarn.resourcemanager.store.class must be specified.
yarn.resourcemanager.recovery.enabled
false
@@ -661,6 +698,16 @@
+
+ The value specifies the file system (e.g. HDFS) path where ResourceManager
+ loads configuration if yarn.resourcemanager.configuration.provider-class
+ is set to org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider.
+
+ yarn.resourcemanager.configuration.file-system-based-store
+ /yarn/conf
+
+
+
The setting that controls whether yarn system metrics is
published on the timeline server or not by RM.
yarn.resourcemanager.system-metrics-publisher.enabled
@@ -683,6 +730,119 @@
+
+
+
+ RM DelegationTokenRenewer thread count
+
+ yarn.resourcemanager.delegation-token-renewer.thread-count
+ 50
+
+
+
+
+ RM secret key update interval in ms
+
+ yarn.resourcemanager.delegation.key.update-interval
+ 86400000
+
+
+
+
+ RM delegation token maximum lifetime in ms
+
+ yarn.resourcemanager.delegation.token.max-lifetime
+ 604800000
+
+
+
+
+ RM delegation token update interval in ms
+
+ yarn.resourcemanager.delegation.token.renew-interval
+ 86400000
+
+
+
+
+ Thread pool size for RMApplicationHistoryWriter.
+
+ yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size
+ 10
+
+
+
+
+ Comma-separated list of values (in minutes) for schedule queue related
+ metrics.
+
+ yarn.resourcemanager.metrics.runtime.buckets
+ 60,300,1440
+
+
+
+
+ Interval for the roll over for the master key used to generate
+ NodeManager tokens. It is expected to be set to a value much larger
+ than yarn.nm.liveness-monitor.expiry-interval-ms.
+
+ yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs
+ 86400
+
+
+
+
+ Flag to enable the ResourceManager reservation system.
+
+ yarn.resourcemanager.reservation-system.enable
+ false
+
+
+
+
+ The Java class to use as the ResourceManager reservation system.
+ By default, is set to
+ org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityReservationSystem
+ when using CapacityScheduler and is set to
+ org.apache.hadoop.yarn.server.resourcemanager.reservation.FairReservationSystem
+ when using FairScheduler.
+
+ yarn.resourcemanager.reservation-system.class
+
+
+
+
+
+ The plan follower policy class name to use for the ResourceManager
+ reservation system.
+ By default, is set to
+ org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacitySchedulerPlanFollower
+ is used when using CapacityScheduler, and is set to
+ org.apache.hadoop.yarn.server.resourcemanager.reservation.FairSchedulerPlanFollower
+ when using FairScheduler.
+
+ yarn.resourcemanager.reservation-system.plan.follower
+
+
+
+
+
+ Step size of the reservation system in ms
+
+ yarn.resourcemanager.reservation-system.planfollower.time-step
+ 1000
+
+
+
+
+ The expiry interval for a container
+
+ yarn.resourcemanager.rm.container-allocation.expiry-interval-ms
+ 600000
+
+
+
+
The hostname of the NM.
yarn.nodemanager.hostname
@@ -828,6 +988,13 @@
+
+ yarn.nodemanager.container-localizer.java.opts
+ -Xmx256m
+
+
+
+
Where to store container logs. An application's localized log directory
will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
Individual containers' log directories will be below this, in directories
@@ -868,6 +1035,17 @@
+
+ How long for ResourceManager to wait for NodeManager to report its
+ log aggregation status. If waiting time of which the log aggregation
+ status is reported from NodeManager exceeds the configured value, RM
+ will report log aggregation status for this NodeManager as TIME_OUT
+
+ yarn.log-aggregation-status.time-out.ms
+ 60000
+
+
+
Time in seconds to retain user logs. Only applicable if
log aggregation is disabled
@@ -945,6 +1123,32 @@
+
+ The https adddress of the NM web application.
+
+ yarn.nodemanager.webapp.https.address
+
+
+
+
+
+ The Kerberos keytab file to be used for spnego filter for the NM web
+ interface.
+
+ yarn.nodemanager.webapp.spnego-keytab-file
+
+
+
+
+
+ The Kerberos principal to be used for spnego filter for the NM web
+ interface.
+
+ yarn.nodemanager.webapp.spnego-principal
+
+
+
+
How often to monitor containers.
yarn.nodemanager.container-monitor.interval-ms
3000
@@ -1052,6 +1256,12 @@
+ Delay in ms between attempts to remove linux cgroup
+ yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms
+ 20
+
+
+
This determines which of the two modes that LCE should use on
a non-secure cluster. If this value is set to true, then all containers
will be launched as the user specified in
@@ -1105,6 +1315,23 @@
+
+ Interval of time the linux container executor should try cleaning up
+ cgroups entry when cleaning up a container.
+
+ yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms
+ 1000
+
+
+
+
+ The UNIX group that the linux-container-executor should run as.
+
+ yarn.nodemanager.linux-container-executor.group
+
+
+
+
T-file compression types used to compress aggregated logs.
yarn.nodemanager.log-aggregation.compression-type
none
@@ -1212,6 +1439,66 @@
+
+ Adjustment to the container OS scheduling priority. In Linux, passed
+ directly to the nice command.
+
+ yarn.nodemanager.container-executor.os.sched.priority.adjustment
+ 0
+
+
+
+
+ Flag to enable container metrics
+
+ yarn.nodemanager.container-metrics.enable
+ true
+
+
+
+
+ Container metrics flush period in ms. Set to -1 for flush on completion.
+
+ yarn.nodemanager.container-metrics.period-ms
+ -1
+
+
+
+
+ Class used to calculate current container resource utilization.
+
+ yarn.nodemanager.container-monitor.process-tree.class
+
+
+
+
+
+ Flag to enable NodeManager disk health checker
+
+ yarn.nodemanager.disk-health-checker.enable
+ true
+
+
+
+
+ Number of threads to use in NM log cleanup. Used when log aggregation
+ is disabled.
+
+ yarn.nodemanager.log.deletion-threads-count
+ 4
+
+
+
+
+ The Windows group that the windows-container-executor should run as.
+
+ yarn.nodemanager.windows-secure-container-executor.group
+
+
+
+
+
+
yarn.nodemanager.docker-container-executor.exec-name
/usr/bin/docker
@@ -1219,24 +1506,28 @@
-
- yarn.nodemanager.aux-services.mapreduce_shuffle.class
- org.apache.hadoop.mapred.ShuffleHandler
+
+ The Docker image name to use for DockerContainerExecutor
+
+ yarn.nodemanager.docker-container-executor.image-name
+
-
- mapreduce.job.jar
-
-
+
mapreduce.job.hdfs-servers
${fs.defaultFS}
-
-
+
+ yarn.nodemanager.aux-services.mapreduce_shuffle.class
+ org.apache.hadoop.mapred.ShuffleHandler
+
+
+
+
The kerberos principal for the proxy, if the proxy is not
running as part of the RM.
@@ -1257,8 +1548,8 @@
-
-
+
+
CLASSPATH for YARN applications. A comma-separated list
@@ -1285,7 +1576,16 @@
-
+
+
+ If true, puts entries from the distributed cache ahead of entries from
+ the default YARN classpath.
+
+ yarn.application.classpath.prepend.distcache
+ false
+
+
+
Indicate to clients whether timeline service is enabled or not.
@@ -1460,6 +1760,7 @@
+
Whether the shared cache is enabled
yarn.sharedcache.enabled
@@ -1601,7 +1902,96 @@
20
-
+
+
+ ACL protocol for use in the Timeline server.
+
+ security.applicationhistory.protocol.acl
+
+
+
+
+
+
+
+ Set to true for MiniYARNCluster unit tests
+
+ yarn.is.minicluster
+ false
+
+
+
+
+ Set for MiniYARNCluster unit tests to control resource monitoring
+
+ yarn.minicluster.control-resource-monitoring
+ false
+
+
+
+
+ Set to false in order to allow MiniYARNCluster to run tests without
+ port conflicts.
+
+ yarn.minicluster.fixed.ports
+ false
+
+
+
+
+ Set to false in order to allow the NodeManager in MiniYARNCluster to
+ use RPC to talk to the RM.
+
+ yarn.minicluster.use-rpc
+ false
+
+
+
+
+ As yarn.nodemanager.resource.memory-mb property but for the NodeManager
+ in a MiniYARNCluster.
+
+ yarn.minicluster.yarn.nodemanager.resource.memory-mb
+ 4096
+
+
+
+
+
+
+ Enable node labels feature
+
+ yarn.node-labels.enabled
+ false
+
+
+
+
+
+ yarn.node-labels.fs-store.retry-policy-spec
+ 2000,500
+
+
+
+
+ URI for NodeLabelManager. The default value is
+ /tmp/hadoop-yarn-${user}/node-labels/ in the local filesystem.
+
+ yarn.node-labels.fs-store.root-dir
+
+
+
+
+
+ Set configuration type for node labels. Administrators can specify
+ "centralized" or "distributed".
+
+ yarn.node-labels.configuration-type
+ centralized
+
+
+
+
The interval that the yarn client library uses to poll the
completion status of the asynchronous API of application client protocol.
@@ -1611,7 +2001,16 @@
- RSS usage of a process computed via
+
+ The duration (in ms) the YARN client waits for an expected state change
+ to occur. -1 means unlimited wait time.
+
+ yarn.client.application-client-protocol.poll-timeout-ms
+ -1
+
+
+
+ RSS usage of a process computed via
/proc/pid/stat is not very accurate as it includes shared pages of a
process. /proc/pid/smaps provides useful information like
Private_Dirty, Private_Clean, Shared_Dirty, Shared_Clean which can be used
@@ -1624,6 +2023,157 @@
+
+ URL for log aggregation server
+
+ yarn.log.server.url
+
+
+
+
+
+ RM Application Tracking URL
+
+ yarn.tracking.url.generator
+
+
+
+
+
+ Class to be used for YarnAuthorizationProvider
+
+ yarn.authorization-provider
+
+
+
+
+
+
+
+ Is the registry enabled: does the RM start it up,
+ create the user and system paths, and purge
+ service records when containers, application attempts
+ and applications complete
+
+ hadoop.registry.rm.enabled
+ false
+
+
+
+
+ Root path in the ZK tree for the registry
+
+ hadoop.registry.zk.root
+ /registry
+
+
+
+
+ Zookeeper session timeout in milliseconds
+
+ hadoop.registry.zk.session.timeout.ms
+ 60000
+
+
+
+
+ Zookeeper session timeout in milliseconds
+
+ hadoop.registry.zk.connection.timeout.ms
+ 15000
+
+
+
+
+ Zookeeper connection retry count before failing
+
+ hadoop.registry.zk.retry.times
+ 5
+
+
+
+
+ Zookeeper connect interval in milliseconds
+
+ hadoop.registry.zk.retry.interval.ms
+ 1000
+
+
+
+
+ Zookeeper retry limit in milliseconds, during
+ exponential backoff: {@value}
+
+ This places a limit even
+ if the retry times and interval limit, combined
+ with the backoff policy, result in a long retry
+ period
+
+ hadoop.registry.zk.retry.ceiling.ms
+ 60000
+
+
+
+
+ List of hostname:port pairs defining the
+ zookeeper quorum binding for the registry
+
+ hadoop.registry.zk.quorum
+ localhost:2181
+
+
+
+
+ Key to set if the registry is secure. Turning it on
+ changes the permissions policy from "open access"
+ to restrictions on kerberos with the option of
+ a user adding one or more auth key pairs down their
+ own tree.
+
+ hadoop.registry.secure
+ false
+
+
+
+
+ A comma separated list of Zookeeper ACL identifiers with
+ system access to the registry in a secure cluster.
+
+ These are given full access to all entries.
+
+ If there is an "@" at the end of a SASL entry it
+ instructs the registry client to append the default kerberos domain.
+
+ hadoop.registry.system.acls
+ sasl:yarn@, sasl:mapred@, sasl:mapred@hdfs@
+
+
+
+
+ The kerberos realm: used to set the realm of
+ system principals which do not declare their realm,
+ and any other accounts that need the value.
+
+ If empty, the default realm of the running process
+ is used.
+
+ If neither are known and the realm is needed, then the registry
+ service/client will fail.
+
+ hadoop.registry.kerberos.realm
+
+
+
+
+
+ Key to define the JAAS context. Used in secure
+ mode
+
+ hadoop.registry.jaas.context
+ Client
+
+
+
Defines how often NMs wake up to upload log files.
The default value is -1. By default, the logs will be uploaded when
the application is finished. By setting this configure, logs can be uploaded
@@ -1633,4 +2183,5 @@
yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds
-1
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 9075d9f6dd3e589399b1c6d46ab40f77f181fa3b..5cbb65388a828ec085c97018e94efac2c7f91e27 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -37,14 +37,14 @@
public void initializeMemberVariables() {
xmlFilename = new String("yarn-default.xml");
configurationClasses = new Class[] { YarnConfiguration.class };
-
// Allocate for usage
configurationPropsToSkipCompare = new HashSet();
+ configurationPrefixToSkipCompare = new HashSet();
// Set error modes
errorIfMissingConfigProps = true;
- errorIfMissingXmlProps = false;
+ errorIfMissingXmlProps = true;
// Specific properties to skip
configurationPropsToSkipCompare
@@ -79,6 +79,37 @@ public void initializeMemberVariables() {
configurationPropsToSkipCompare
.add(YarnConfiguration
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_SCM_STORE_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
+
+ // Ignore all YARN Application Timeline Service (version 1) properties
+ configurationPrefixToSkipCompare.add("yarn.timeline-service.");
+
+ // Used as Java command line properties, not XML
+ configurationPrefixToSkipCompare.add("yarn.app.container");
+
+ // Ignore NodeManager "work in progress" variables
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED);
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_NETWORK_RESOURCE_INTERFACE);
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT);
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT);
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.NM_DISK_RESOURCE_ENABLED);
+
+ // Set by container-executor.cfg
+ configurationPrefixToSkipCompare.add(YarnConfiguration.NM_USER_HOME_DIR);
+
+ // Ignore deprecated properties
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS);
// Allocate for usage
xmlPropsToSkipCompare = new HashSet();
@@ -94,13 +125,11 @@ public void initializeMemberVariables() {
// Used in the XML file as a variable reference internal to the XML file
xmlPropsToSkipCompare.add("yarn.nodemanager.hostname");
- xmlPropsToSkipCompare.add("yarn.timeline-service.hostname");
- // Currently defined in TimelineAuthenticationFilterInitializer
- xmlPrefixToSkipCompare.add("yarn.timeline-service.http-authentication");
+ // Ignore all YARN Application Timeline Service (version 1) properties
+ xmlPrefixToSkipCompare.add("yarn.timeline-service");
// Currently defined in RegistryConstants
xmlPrefixToSkipCompare.add("hadoop.registry");
}
-
}