diff --git hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md index f9647359d736bcb8a036820577a7a48426d7aab2..f8bc95b9a5479ca5d84890794df475985e5c168f 100644 --- hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md +++ hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md @@ -276,6 +276,7 @@ The following table lists the configuration property names that are deprecated i | user.name | mapreduce.job.user.name | | webinterface.private.actions | mapreduce.jobtracker.webinterface.trusted | | yarn.app.mapreduce.yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | +| yarn.client.app-submission.poll-interval | yarn.client.application-client-protocol.poll-timeout-ms | The following table lists additional changes to some configuration properties: diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index d94157cfed6ece3f000647ab61f439f72ee5511f..c5072639fc26639413f0d3e4aa3e6cc2caa01d9c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -24,7 +24,8 @@ - + + Factory to create client IPC classes. yarn.ipc.client.factory.class @@ -46,7 +47,8 @@ org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC - + + The hostname of the RM. yarn.resourcemanager.hostname @@ -131,6 +133,32 @@ + + The Kerberos keytab file to be used for spnego filter for the RM web + interface. + + yarn.resourcemanager.webapp.spnego-keytab-file + + + + + + The Kerberos principal to be used for spnego filter for the RM web + interface. + + yarn.resourcemanager.webapp.spnego-principal + + + + + + Add button to kill application in the RM Application view. + + yarn.resourcemanager.webapp.ui-actions.enabled + true + + + yarn.resourcemanager.resource-tracker.address ${yarn.resourcemanager.hostname}:8031 @@ -268,7 +296,16 @@ - Enable RM to recover state after starting. If true, then + + Used by node labels. If set to true, the port should be included in the + node name. Only usable if your scheduler supports node labels. + + yarn.scheduler.include-port-in-node-name + false + + + + Enable RM to recover state after starting. If true, then yarn.resourcemanager.store.class must be specified. yarn.resourcemanager.recovery.enabled false @@ -424,7 +461,7 @@ average, the following n1 retries sleep t1 milliseconds on average, and so on. yarn.resourcemanager.fs.state-store.retry-policy-spec - 2000, 500 + 2000,500 @@ -661,6 +698,16 @@ + + The value specifies the file system (e.g. HDFS) path where ResourceManager + loads configuration if yarn.resourcemanager.configuration.provider-class + is set to org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider. + + yarn.resourcemanager.configuration.file-system-based-store + /yarn/conf + + + The setting that controls whether yarn system metrics is published on the timeline server or not by RM. yarn.resourcemanager.system-metrics-publisher.enabled @@ -683,6 +730,119 @@ + + + + RM DelegationTokenRenewer thread count + + yarn.resourcemanager.delegation-token-renewer.thread-count + 50 + + + + + RM secret key update interval in ms + + yarn.resourcemanager.delegation.key.update-interval + 86400000 + + + + + RM delegation token maximum lifetime in ms + + yarn.resourcemanager.delegation.token.max-lifetime + 604800000 + + + + + RM delegation token update interval in ms + + yarn.resourcemanager.delegation.token.renew-interval + 86400000 + + + + + Thread pool size for RMApplicationHistoryWriter. + + yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size + 10 + + + + + Comma-separated list of values (in minutes) for schedule queue related + metrics. + + yarn.resourcemanager.metrics.runtime.buckets + 60,300,1440 + + + + + Interval for the roll over for the master key used to generate + NodeManager tokens. It is expected to be set to a value much larger + than yarn.nm.liveness-monitor.expiry-interval-ms. + + yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs + 86400 + + + + + Flag to enable the ResourceManager reservation system. + + yarn.resourcemanager.reservation-system.enable + false + + + + + The Java class to use as the ResourceManager reservation system. + By default, is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityReservationSystem + when using CapacityScheduler and is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.FairReservationSystem + when using FairScheduler. + + yarn.resourcemanager.reservation-system.class + + + + + + The plan follower policy class name to use for the ResourceManager + reservation system. + By default, is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacitySchedulerPlanFollower + is used when using CapacityScheduler, and is set to + org.apache.hadoop.yarn.server.resourcemanager.reservation.FairSchedulerPlanFollower + when using FairScheduler. + + yarn.resourcemanager.reservation-system.plan.follower + + + + + + Step size of the reservation system in ms + + yarn.resourcemanager.reservation-system.planfollower.time-step + 1000 + + + + + The expiry interval for a container + + yarn.resourcemanager.rm.container-allocation.expiry-interval-ms + 600000 + + + + The hostname of the NM. yarn.nodemanager.hostname @@ -828,7 +988,14 @@ - Where to store container logs. An application's localized log directory + + yarn.nodemanager.container-localizer.java.opts + -Xmx256m + + + + + Where to store container logs. An application's localized log directory will be found in ${yarn.nodemanager.log-dirs}/application_${appid}. Individual containers' log directories will be below this, in directories named container_{$contid}. Each container directory will contain the files @@ -868,6 +1035,17 @@ + + How long for ResourceManager to wait for NodeManager to report its + log aggregation status. If waiting time of which the log aggregation + status is reported from NodeManager exceeds the configured value, RM + will report log aggregation status for this NodeManager as TIME_OUT + + yarn.log-aggregation-status.time-out.ms + 60000 + + + Time in seconds to retain user logs. Only applicable if log aggregation is disabled @@ -994,6 +1172,32 @@ + + The https adddress of the NM web application. + + yarn.nodemanager.webapp.https.address + + + + + + The Kerberos keytab file to be used for spnego filter for the NM web + interface. + + yarn.nodemanager.webapp.spnego-keytab-file + + + + + + The Kerberos principal to be used for spnego filter for the NM web + interface. + + yarn.nodemanager.webapp.spnego-principal + + + + How often to monitor containers. yarn.nodemanager.container-monitor.interval-ms 3000 @@ -1101,6 +1305,12 @@ + Delay in ms between attempts to remove linux cgroup + yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms + 20 + + + This determines which of the two modes that LCE should use on a non-secure cluster. If this value is set to true, then all containers will be launched as the user specified in @@ -1154,6 +1364,23 @@ + + Interval of time the linux container executor should try cleaning up + cgroups entry when cleaning up a container. + + yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms + 1000 + + + + + The UNIX group that the linux-container-executor should run as. + + yarn.nodemanager.linux-container-executor.group + + + + T-file compression types used to compress aggregated logs. yarn.nodemanager.log-aggregation.compression-type none @@ -1261,6 +1488,66 @@ + + Adjustment to the container OS scheduling priority. In Linux, passed + directly to the nice command. + + yarn.nodemanager.container-executor.os.sched.priority.adjustment + 0 + + + + + Flag to enable container metrics + + yarn.nodemanager.container-metrics.enable + true + + + + + Container metrics flush period in ms. Set to -1 for flush on completion. + + yarn.nodemanager.container-metrics.period-ms + -1 + + + + + Class used to calculate current container resource utilization. + + yarn.nodemanager.container-monitor.process-tree.class + + + + + + Flag to enable NodeManager disk health checker + + yarn.nodemanager.disk-health-checker.enable + true + + + + + Number of threads to use in NM log cleanup. Used when log aggregation + is disabled. + + yarn.nodemanager.log.deletion-threads-count + 4 + + + + + The Windows group that the windows-container-executor should run as. + + yarn.nodemanager.windows-secure-container-executor.group + + + + + + yarn.nodemanager.docker-container-executor.exec-name /usr/bin/docker @@ -1268,24 +1555,28 @@ - - yarn.nodemanager.aux-services.mapreduce_shuffle.class - org.apache.hadoop.mapred.ShuffleHandler + + The Docker image name to use for DockerContainerExecutor + + yarn.nodemanager.docker-container-executor.image-name + - - mapreduce.job.jar - - + mapreduce.job.hdfs-servers ${fs.defaultFS} - - + + yarn.nodemanager.aux-services.mapreduce_shuffle.class + org.apache.hadoop.mapred.ShuffleHandler + + + + The kerberos principal for the proxy, if the proxy is not running as part of the RM. @@ -1306,8 +1597,8 @@ - - + + CLASSPATH for YARN applications. A comma-separated list @@ -1334,7 +1625,7 @@ - + Indicate to clients whether timeline service is enabled or not. @@ -1518,6 +1809,7 @@ + Whether the shared cache is enabled yarn.sharedcache.enabled @@ -1659,7 +1951,99 @@ 20 - + + + ACL protocol for use in the Timeline server. + + security.applicationhistory.protocol.acl + + + + + + + + Set to true for MiniYARNCluster unit tests + + yarn.is.minicluster + false + + + + + Set for MiniYARNCluster unit tests to control resource monitoring + + yarn.minicluster.control-resource-monitoring + false + + + + + Set to false in order to allow MiniYARNCluster to run tests without + port conflicts. + + yarn.minicluster.fixed.ports + false + + + + + Set to false in order to allow the NodeManager in MiniYARNCluster to + use RPC to talk to the RM. + + yarn.minicluster.use-rpc + false + + + + + As yarn.nodemanager.resource.memory-mb property but for the NodeManager + in a MiniYARNCluster. + + yarn.minicluster.yarn.nodemanager.resource.memory-mb + 4096 + + + + + + + Enable node labels feature + + yarn.node-labels.enabled + false + + + + + Retry policy used for FileSystem node label store. The policy is + specified by N pairs of sleep-time in milliseconds and number-of-retries + "s1,n1,s2,n2,...". + + yarn.node-labels.fs-store.retry-policy-spec + 2000,500 + + + + + URI for NodeLabelManager. The default value is + /tmp/hadoop-yarn-${user}/node-labels/ in the local filesystem. + + yarn.node-labels.fs-store.root-dir + + + + + + Set configuration type for node labels. Administrators can specify + "centralized" or "distributed". + + yarn.node-labels.configuration-type + centralized + + + + The interval that the yarn client library uses to poll the completion status of the asynchronous API of application client protocol. @@ -1669,7 +2053,16 @@ - RSS usage of a process computed via + + The duration (in ms) the YARN client waits for an expected state change + to occur. -1 means unlimited wait time. + + yarn.client.application-client-protocol.poll-timeout-ms + -1 + + + + RSS usage of a process computed via /proc/pid/stat is not very accurate as it includes shared pages of a process. /proc/pid/smaps provides useful information like Private_Dirty, Private_Clean, Shared_Dirty, Shared_Clean which can be used @@ -1682,6 +2075,30 @@ + + URL for log aggregation server + + yarn.log.server.url + + + + + + RM Application Tracking URL + + yarn.tracking.url.generator + + + + + + Class to be used for YarnAuthorizationProvider + + yarn.authorization-provider + + + + Defines how often NMs wake up to upload log files. The default value is -1. By default, the logs will be uploaded when the application is finished. By setting this configure, logs can be uploaded @@ -1691,4 +2108,15 @@ yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds -1 + + + + Enable/disable intermediate-data encryption at YARN level. For now, + this only is used by the FileSystemRMStateStore to setup right + file-system security attributes. + + yarn.intermediate-data-encryption.enable + + + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 9075d9f6dd3e589399b1c6d46ab40f77f181fa3b..10e1ca6de665c17fb6e33d7f5476cb933bb36bac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -33,18 +33,19 @@ */ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { + @SuppressWarnings("deprecation") @Override public void initializeMemberVariables() { xmlFilename = new String("yarn-default.xml"); configurationClasses = new Class[] { YarnConfiguration.class }; - // Allocate for usage configurationPropsToSkipCompare = new HashSet(); + configurationPrefixToSkipCompare = new HashSet(); // Set error modes errorIfMissingConfigProps = true; - errorIfMissingXmlProps = false; + errorIfMissingXmlProps = true; // Specific properties to skip configurationPropsToSkipCompare @@ -79,6 +80,37 @@ public void initializeMemberVariables() { configurationPropsToSkipCompare .add(YarnConfiguration .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SCM_STORE_CLASS); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS); + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL); + + // Ignore all YARN Application Timeline Service (version 1) properties + configurationPrefixToSkipCompare.add("yarn.timeline-service."); + + // Used as Java command line properties, not XML + configurationPrefixToSkipCompare.add("yarn.app.container"); + + // Ignore NodeManager "work in progress" variables + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_INTERFACE); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT); + configurationPrefixToSkipCompare + .add(YarnConfiguration.NM_DISK_RESOURCE_ENABLED); + + // Set by container-executor.cfg + configurationPrefixToSkipCompare.add(YarnConfiguration.NM_USER_HOME_DIR); + + // Ignore deprecated properties + configurationPrefixToSkipCompare + .add(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS); // Allocate for usage xmlPropsToSkipCompare = new HashSet(); @@ -94,13 +126,11 @@ public void initializeMemberVariables() { // Used in the XML file as a variable reference internal to the XML file xmlPropsToSkipCompare.add("yarn.nodemanager.hostname"); - xmlPropsToSkipCompare.add("yarn.timeline-service.hostname"); - // Currently defined in TimelineAuthenticationFilterInitializer - xmlPrefixToSkipCompare.add("yarn.timeline-service.http-authentication"); + // Ignore all YARN Application Timeline Service (version 1) properties + xmlPrefixToSkipCompare.add("yarn.timeline-service"); - // Currently defined in RegistryConstants + // Currently defined in RegistryConstants/core-site.xml xmlPrefixToSkipCompare.add("hadoop.registry"); } - }