diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/YarnException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/YarnException.java new file mode 100644 index 0000000..b12fc81 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/YarnException.java @@ -0,0 +1,29 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn; + +/** Base Yarn Exception. + */ +public class YarnException extends RuntimeException { + public YarnException(Throwable cause) { super(cause); } + public YarnException(String message) { super(message); } + public YarnException(String message, Throwable cause) { + super(message, cause); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java new file mode 100644 index 0000000..de8b6d1 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -0,0 +1,757 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.conf; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.Arrays; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.ApplicationConstants; + +import com.google.common.base.Joiner; + +public class YarnConfiguration extends Configuration { + + private static final Joiner JOINER = Joiner.on(""); + + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; + private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; + + static { + Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE); + Configuration.addDefaultResource(YARN_SITE_XML_FILE); + } + + //Configurations + + public static final String YARN_PREFIX = "yarn."; + + /** Delay before deleting resource to ease debugging of NM issues */ + public static final String DEBUG_NM_DELETE_DELAY_SEC = + YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; + + //////////////////////////////// + // IPC Configs + //////////////////////////////// + public static final String IPC_PREFIX = YARN_PREFIX + "ipc."; + + /** Factory to create client IPC classes.*/ + public static final String IPC_CLIENT_FACTORY = + IPC_PREFIX + "client.factory.class"; + + /** Type of serialization to use.*/ + public static final String IPC_SERIALIZER_TYPE = + IPC_PREFIX + "serializer.type"; + public static final String DEFAULT_IPC_SERIALIZER_TYPE = "protocolbuffers"; + + /** Factory to create server IPC classes.*/ + public static final String IPC_SERVER_FACTORY = + IPC_PREFIX + "server.factory.class"; + + /** Factory to create IPC exceptions.*/ + public static final String IPC_EXCEPTION_FACTORY = + IPC_PREFIX + "exception.factory.class"; + + /** Factory to create serializeable records.*/ + public static final String IPC_RECORD_FACTORY_CLASS = + IPC_PREFIX + "record.factory.class"; + public static final String DEFAULT_IPC_RECORD_FACTORY_CLASS = + "org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl"; + + /** RPC class implementation*/ + public static final String IPC_RPC_IMPL = + IPC_PREFIX + "rpc.class"; + public static final String DEFAULT_IPC_RPC_IMPL = + "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC"; + + //////////////////////////////// + // Resource Manager Configs + //////////////////////////////// + public static final String RM_PREFIX = "yarn.resourcemanager."; + + /** The address of the applications manager interface in the RM.*/ + public static final String RM_ADDRESS = + RM_PREFIX + "address"; + public static final int DEFAULT_RM_PORT = 8032; + public static final String DEFAULT_RM_ADDRESS = + "0.0.0.0:" + DEFAULT_RM_PORT; + + /** The number of threads used to handle applications manager requests.*/ + public static final String RM_CLIENT_THREAD_COUNT = + RM_PREFIX + "client.thread-count"; + public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50; + + /** The Kerberos principal for the resource manager.*/ + public static final String RM_PRINCIPAL = + RM_PREFIX + "principal"; + + /** The address of the scheduler interface.*/ + public static final String RM_SCHEDULER_ADDRESS = + RM_PREFIX + "scheduler.address"; + public static final int DEFAULT_RM_SCHEDULER_PORT = 8030; + public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" + + DEFAULT_RM_SCHEDULER_PORT; + + /** Miniumum request grant-able by the RM scheduler. */ + public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB = + YARN_PREFIX + "scheduler.minimum-allocation-mb"; + public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; + public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = + YARN_PREFIX + "scheduler.minimum-allocation-vcores"; + public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; + + /** Maximum request grant-able by the RM scheduler. */ + public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = + YARN_PREFIX + "scheduler.maximum-allocation-mb"; + public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192; + public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = + YARN_PREFIX + "scheduler.maximum-allocation-vcores"; + public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; + + /** Number of threads to handle scheduler interface.*/ + public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = + RM_PREFIX + "scheduler.client.thread-count"; + public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; + + /** The address of the RM web application.*/ + public static final String RM_WEBAPP_ADDRESS = + RM_PREFIX + "webapp.address"; + + public static final int DEFAULT_RM_WEBAPP_PORT = 8088; + public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" + + DEFAULT_RM_WEBAPP_PORT; + + public static final String RM_RESOURCE_TRACKER_ADDRESS = + RM_PREFIX + "resource-tracker.address"; + public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031; + public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS = + "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT; + + /** The expiry interval for application master reporting.*/ + public static final String RM_AM_EXPIRY_INTERVAL_MS = + YARN_PREFIX + "am.liveness-monitor.expiry-interval-ms"; + public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000; + + /** How long to wait until a node manager is considered dead.*/ + public static final String RM_NM_EXPIRY_INTERVAL_MS = + YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms"; + public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000; + + /** Are acls enabled.*/ + public static final String YARN_ACL_ENABLE = + YARN_PREFIX + "acl.enable"; + public static final boolean DEFAULT_YARN_ACL_ENABLE = false; + + /** ACL of who can be admin of YARN cluster.*/ + public static final String YARN_ADMIN_ACL = + YARN_PREFIX + "admin.acl"; + public static final String DEFAULT_YARN_ADMIN_ACL = "*"; + + /** ACL used in case none is found. Allows nothing. */ + public static final String DEFAULT_YARN_APP_ACL = " "; + + /** The address of the RM admin interface.*/ + public static final String RM_ADMIN_ADDRESS = + RM_PREFIX + "admin.address"; + public static final int DEFAULT_RM_ADMIN_PORT = 8033; + public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" + + DEFAULT_RM_ADMIN_PORT; + + /**Number of threads used to handle RM admin interface.*/ + public static final String RM_ADMIN_CLIENT_THREAD_COUNT = + RM_PREFIX + "admin.client.thread-count"; + public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1; + + /** + * The maximum number of application attempts. + * It's a global setting for all application masters. + */ + public static final String RM_AM_MAX_ATTEMPTS = + RM_PREFIX + "am.max-attempts"; + public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2; + + /** The keytab for the resource manager.*/ + public static final String RM_KEYTAB = + RM_PREFIX + "keytab"; + + /** How long to wait until a container is considered dead.*/ + public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = + RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; + public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000; + + /** Path to file with nodes to include.*/ + public static final String RM_NODES_INCLUDE_FILE_PATH = + RM_PREFIX + "nodes.include-path"; + public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = ""; + + /** Path to file with nodes to exclude.*/ + public static final String RM_NODES_EXCLUDE_FILE_PATH = + RM_PREFIX + "nodes.exclude-path"; + public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = ""; + + /** Number of threads to handle resource tracker calls.*/ + public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = + RM_PREFIX + "resource-tracker.client.thread-count"; + public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50; + + /** The class to use as the resource scheduler.*/ + public static final String RM_SCHEDULER = + RM_PREFIX + "scheduler.class"; + + public static final String DEFAULT_RM_SCHEDULER = + "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"; + + /** RM set next Heartbeat interval for NM */ + public static final String RM_NM_HEARTBEAT_INTERVAL_MS = + RM_PREFIX + "nodemanagers.heartbeat-interval-ms"; + public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000; + + //Delegation token related keys + public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = + RM_PREFIX + "delegation.key.update-interval"; + public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = + 24*60*60*1000; // 1 day + public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY = + RM_PREFIX + "delegation.token.renew-interval"; + public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = + 24*60*60*1000; // 1 day + public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY = + RM_PREFIX + "delegation.token.max-lifetime"; + public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = + 7*24*60*60*1000; // 7 days + + public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled"; + public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false; + + /** The class to use as the persistent store.*/ + public static final String RM_STORE = RM_PREFIX + "store.class"; + + /** URI for FileSystemRMStateStore */ + public static final String FS_RM_STATE_STORE_URI = + RM_PREFIX + "fs.rm-state-store.uri"; + + /** The maximum number of completed applications RM keeps. */ + public static final String RM_MAX_COMPLETED_APPLICATIONS = + RM_PREFIX + "max-completed-applications"; + public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000; + + /** Default application name */ + public static final String DEFAULT_APPLICATION_NAME = "N/A"; + + /** Default queue name */ + public static final String DEFAULT_QUEUE_NAME = "default"; + + /** + * Buckets (in minutes) for the number of apps running in each queue. + */ + public static final String RM_METRICS_RUNTIME_BUCKETS = + RM_PREFIX + "metrics.runtime.buckets"; + + /** + * Default sizes of the runtime metric buckets in minutes. + */ + public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = + "60,300,1440"; + + public static final String RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX + + "application-tokens.master-key-rolling-interval-secs"; + + public static final long DEFAULT_RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = + 24 * 60 * 60; + + public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = + RM_PREFIX + "container-tokens.master-key-rolling-interval-secs"; + + public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = + 24 * 60 * 60; + + //////////////////////////////// + // Node Manager Configs + //////////////////////////////// + + /** Prefix for all node manager configs.*/ + public static final String NM_PREFIX = "yarn.nodemanager."; + + /** Environment variables that will be sent to containers.*/ + public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env"; + public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX"; + + /** Environment variables that containers may override rather than use NodeManager's default.*/ + public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist"; + public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",", + Arrays.asList(ApplicationConstants.Environment.JAVA_HOME.key(), + ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(), + ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(), + ApplicationConstants.Environment.HADOOP_CONF_DIR.key(), + ApplicationConstants.Environment.HADOOP_YARN_HOME.key())); + + /** address of node manager IPC.*/ + public static final String NM_ADDRESS = NM_PREFIX + "address"; + public static final int DEFAULT_NM_PORT = 0; + public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:" + + DEFAULT_NM_PORT; + + /** who will execute(launch) the containers.*/ + public static final String NM_CONTAINER_EXECUTOR = + NM_PREFIX + "container-executor.class"; + + /** + * Adjustment to make to the container os scheduling priority. + * The valid values for this could vary depending on the platform. + * On Linux, higher values mean run the containers at a less + * favorable priority than the NM. + * The value specified is an int. + */ + public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = + NM_PREFIX + "container-executor.os.sched.priority.adjustment"; + public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0; + + /** Number of threads container manager uses.*/ + public static final String NM_CONTAINER_MGR_THREAD_COUNT = + NM_PREFIX + "container-manager.thread-count"; + public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; + + /** Number of threads used in cleanup.*/ + public static final String NM_DELETE_THREAD_COUNT = + NM_PREFIX + "delete.thread-count"; + public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4; + + /** Keytab for NM.*/ + public static final String NM_KEYTAB = NM_PREFIX + "keytab"; + + /**List of directories to store localized files in.*/ + public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs"; + public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir"; + + /** + * Number of files in each localized directories + * Avoid tuning this too low. + */ + public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = + NM_PREFIX + "local-cache.max-files-per-directory"; + public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192; + + /** Address where the localizer IPC is.*/ + public static final String NM_LOCALIZER_ADDRESS = + NM_PREFIX + "localizer.address"; + public static final int DEFAULT_NM_LOCALIZER_PORT = 8040; + public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" + + DEFAULT_NM_LOCALIZER_PORT; + + /** Interval in between cache cleanups.*/ + public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = + NM_PREFIX + "localizer.cache.cleanup.interval-ms"; + public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = + 10 * 60 * 1000; + + /** Target size of localizer cache in MB, per local directory.*/ + public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB = + NM_PREFIX + "localizer.cache.target-size-mb"; + public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024; + + /** Number of threads to handle localization requests.*/ + public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT = + NM_PREFIX + "localizer.client.thread-count"; + public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5; + + /** Number of threads to use for localization fetching.*/ + public static final String NM_LOCALIZER_FETCH_THREAD_COUNT = + NM_PREFIX + "localizer.fetch.thread-count"; + public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4; + + /** Where to store container logs.*/ + public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; + public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; + + /** Interval at which the delayed token removal thread runs */ + public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = + RM_PREFIX + "delayed.delegation-token.removal-interval-ms"; + public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = + 30000l; + + /** Whether to enable log aggregation */ + public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX + + "log-aggregation-enable"; + public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false; + + /** + * How long to wait before deleting aggregated logs, -1 disables. + * Be careful set this too small and you will spam the name node. + */ + public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX + + "log-aggregation.retain-seconds"; + public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; + + /** + * How long to wait between aggregated log retention checks. If set to + * a value <= 0 then the value is computed as one-tenth of the log retention + * setting. Be careful set this too small and you will spam the name node. + */ + public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = + YARN_PREFIX + "log-aggregation.retain-check-interval-seconds"; + public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1; + + /** + * Number of seconds to retain logs on the NodeManager. Only applicable if Log + * aggregation is disabled + */ + public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX + + "log.retain-seconds"; + + /** + * Number of threads used in log cleanup. Only applicable if Log aggregation + * is disabled + */ + public static final String NM_LOG_DELETION_THREADS_COUNT = + NM_PREFIX + "log.deletion-threads-count"; + public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4; + + /** Where to aggregate logs to.*/ + public static final String NM_REMOTE_APP_LOG_DIR = + NM_PREFIX + "remote-app-log-dir"; + public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs"; + + /** + * The remote log dir will be created at + * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId} + */ + public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = + NM_PREFIX + "remote-app-log-dir-suffix"; + public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs"; + + public static final String YARN_LOG_SERVER_URL = + YARN_PREFIX + "log.server.url"; + + public static final String YARN_TRACKING_URL_GENERATOR = + YARN_PREFIX + "tracking.url.generator"; + + /** Amount of memory in GB that can be allocated for containers.*/ + public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb"; + public static final int DEFAULT_NM_PMEM_MB = 8 * 1024; + + /** Specifies whether physical memory check is enabled. */ + public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX + + "pmem-check-enabled"; + public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true; + + /** Specifies whether physical memory check is enabled. */ + public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX + + "vmem-check-enabled"; + public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true; + + /** Conversion ratio for physical memory to virtual memory. */ + public static final String NM_VMEM_PMEM_RATIO = + NM_PREFIX + "vmem-pmem-ratio"; + public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f; + + /** Number of Physical CPU Cores which can be allocated for containers.*/ + public static final String NM_VCORES = NM_PREFIX + "resource.cpu-cores"; + public static final int DEFAULT_NM_VCORES = 8; + + /** Conversion ratio for physical cores to virtual cores. */ + public static final String NM_VCORES_PCORES_RATIO = + NM_PREFIX + "vcores-pcores-ratio"; + public static final float DEFAULT_NM_VCORES_PCORES_RATIO = 2.0f; + + /** NM Webapp address.**/ + public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; + public static final int DEFAULT_NM_WEBAPP_PORT = 8042; + public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" + + DEFAULT_NM_WEBAPP_PORT; + + /** How often to monitor containers.*/ + public final static String NM_CONTAINER_MON_INTERVAL_MS = + NM_PREFIX + "container-monitor.interval-ms"; + public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000; + + /** Class that calculates containers current resource utilization.*/ + public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR = + NM_PREFIX + "container-monitor.resource-calculator.class"; + /** Class that calculates process tree resource utilization.*/ + public static final String NM_CONTAINER_MON_PROCESS_TREE = + NM_PREFIX + "container-monitor.process-tree.class"; + + /** + * Enable/Disable disks' health checker. Default is true. + * An expert level configuration property. + */ + public static final String NM_DISK_HEALTH_CHECK_ENABLE = + NM_PREFIX + "disk-health-checker.enable"; + /** Frequency of running disks' health checker.*/ + public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS = + NM_PREFIX + "disk-health-checker.interval-ms"; + /** By default, disks' health is checked every 2 minutes. */ + public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS = + 2 * 60 * 1000; + + /** + * The minimum fraction of number of disks to be healthy for the nodemanager + * to launch new containers. This applies to nm-local-dirs and nm-log-dirs. + */ + public static final String NM_MIN_HEALTHY_DISKS_FRACTION = + NM_PREFIX + "disk-health-checker.min-healthy-disks"; + /** + * By default, at least 5% of disks are to be healthy to say that the node + * is healthy in terms of disks. + */ + public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION + = 0.25F; + + /** Frequency of running node health script.*/ + public static final String NM_HEALTH_CHECK_INTERVAL_MS = + NM_PREFIX + "health-checker.interval-ms"; + public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000; + + /** Health check script time out period.*/ + public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = + NM_PREFIX + "health-checker.script.timeout-ms"; + public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = + 2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS; + + /** The health check script to run.*/ + public static final String NM_HEALTH_CHECK_SCRIPT_PATH = + NM_PREFIX + "health-checker.script.path"; + + /** The arguments to pass to the health check script.*/ + public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = + NM_PREFIX + "health-checker.script.opts"; + + /** The path to the Linux container executor.*/ + public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = + NM_PREFIX + "linux-container-executor.path"; + + /** + * The UNIX group that the linux-container-executor should run as. + * This is intended to be set as part of container-executor.cfg. + */ + public static final String NM_LINUX_CONTAINER_GROUP = + NM_PREFIX + "linux-container-executor.group"; + + /** The type of resource enforcement to use with the + * linux container executor. + */ + public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER = + NM_PREFIX + "linux-container-executor.resources-handler.class"; + + /** The path the linux container executor should use for cgroups */ + public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY = + NM_PREFIX + "linux-container-executor.cgroups.hierarchy"; + + /** Whether the linux container executor should mount cgroups if not found */ + public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT = + NM_PREFIX + "linux-container-executor.cgroups.mount"; + + /** Where the linux container executor should mount cgroups if not found */ + public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = + NM_PREFIX + "linux-container-executor.cgroups.mount-path"; + + /** T-file compression types used to compress aggregated logs.*/ + public static final String NM_LOG_AGG_COMPRESSION_TYPE = + NM_PREFIX + "log-aggregation.compression-type"; + public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none"; + + /** The kerberos principal for the node manager.*/ + public static final String NM_PRINCIPAL = + NM_PREFIX + "principal"; + + public static final String NM_AUX_SERVICES = + NM_PREFIX + "aux-services"; + + public static final String NM_AUX_SERVICE_FMT = + NM_PREFIX + "aux-services.%s.class"; + + public static final String NM_USER_HOME_DIR = + NM_PREFIX + "user-home-dir"; + + public static final String DEFAULT_NM_USER_HOME_DIR= "/home/"; + + //////////////////////////////// + // Web Proxy Configs + //////////////////////////////// + public static final String PROXY_PREFIX = "yarn.web-proxy."; + + /** The kerberos principal for the proxy.*/ + public static final String PROXY_PRINCIPAL = + PROXY_PREFIX + "principal"; + + /** Keytab for Proxy.*/ + public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab"; + + /** The address for the web proxy.*/ + public static final String PROXY_ADDRESS = + PROXY_PREFIX + "address"; + + /** + * YARN Service Level Authorization + */ + public static final String + YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER = + "security.resourcetracker.protocol.acl"; + public static final String + YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER = + "security.client.resourcemanager.protocol.acl"; + public static final String + YARN_SECURITY_SERVICE_AUTHORIZATION_ADMIN = + "security.admin.protocol.acl"; + public static final String + YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_RESOURCEMANAGER = + "security.applicationmaster.resourcemanager.protocol.acl"; + + public static final String + YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGER = + "security.containermanager.protocol.acl"; + public static final String + YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER = + "security.resourcelocalizer.protocol.acl"; + + /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL + * to a running container */ + public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = + NM_PREFIX + "sleep-delay-before-sigkill.ms"; + public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = + 250; + + /** Max time to wait for a process to come up when trying to cleanup + * container resources */ + public static final String NM_PROCESS_KILL_WAIT_MS = + NM_PREFIX + "process-kill-wait.ms"; + public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS = + 2000; + + /** Max time to wait to establish a connection to RM when NM starts + */ + public static final String RESOURCEMANAGER_CONNECT_WAIT_SECS = + NM_PREFIX + "resourcemanager.connect.wait.secs"; + public static final int DEFAULT_RESOURCEMANAGER_CONNECT_WAIT_SECS = + 15*60; + + /** Time interval between each NM attempt to connect to RM + */ + public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS = + NM_PREFIX + "resourcemanager.connect.retry_interval.secs"; + public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS + = 30; + + /** + * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH + * entries + */ + public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX + + "application.classpath"; + + /** + * Default CLASSPATH for YARN applications. A comma-separated list of + * CLASSPATH entries + */ + public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = { + ApplicationConstants.Environment.HADOOP_CONF_DIR.$(), + ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() + + "/share/hadoop/common/*", + ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() + + "/share/hadoop/common/lib/*", + ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() + + "/share/hadoop/hdfs/*", + ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() + + "/share/hadoop/hdfs/lib/*", + ApplicationConstants.Environment.HADOOP_YARN_HOME.$() + + "/share/hadoop/yarn/*", + ApplicationConstants.Environment.HADOOP_YARN_HOME.$() + + "/share/hadoop/yarn/lib/*" }; + + /** Container temp directory */ + public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; + + public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX + + "is.minicluster"; + + /** Whether to use fixed ports with the minicluster. */ + public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX + + "minicluster.fixed.ports"; + + /** + * Default is false to be able to run tests concurrently without port + * conflicts. + */ + public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false; + + //////////////////////////////// + // Other Configs + //////////////////////////////// + + /** + * The interval of the yarn client's querying application state after + * application submission. The unit is millisecond. + */ + public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = + YARN_PREFIX + "client.app-submission.poll-interval"; + public static final long DEFAULT_YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = + 1000; + + public YarnConfiguration() { + super(); + } + + public YarnConfiguration(Configuration conf) { + super(conf); + if (! (conf instanceof YarnConfiguration)) { + this.reloadConfiguration(); + } + } + + public static String getProxyHostAndPort(Configuration conf) { + String addr = conf.get(PROXY_ADDRESS); + if(addr == null || addr.isEmpty()) { + addr = getRMWebAppHostAndPort(conf); + } + return addr; + } + + public static String getRMWebAppHostAndPort(Configuration conf) { + InetSocketAddress address = conf.getSocketAddr( + YarnConfiguration.RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); + address = NetUtils.getConnectAddress(address); + StringBuffer sb = new StringBuffer(); + InetAddress resolved = address.getAddress(); + if (resolved == null || resolved.isAnyLocalAddress() || + resolved.isLoopbackAddress()) { + String lh = address.getHostName(); + try { + lh = InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + //Ignore and fallback. + } + sb.append(lh); + } else { + sb.append(address.getHostName()); + } + sb.append(":").append(address.getPort()); + return sb.toString(); + } + + public static String getRMWebAppURL(Configuration conf) { + return JOINER.join("http://", getRMWebAppHostAndPort(conf)); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factories/RecordFactory.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factories/RecordFactory.java new file mode 100644 index 0000000..e62d5a6 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factories/RecordFactory.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.factories; + +import org.apache.hadoop.yarn.YarnException; + + +public interface RecordFactory { + public T newRecordInstance(Class clazz) throws YarnException; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factory/providers/RecordFactoryProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factory/providers/RecordFactoryProvider.java new file mode 100644 index 0000000..1866b05 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factory/providers/RecordFactoryProvider.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.factory.providers; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.YarnException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; + +public class RecordFactoryProvider { + private static Configuration defaultConf; + + static { + defaultConf = new Configuration(); + } + + private RecordFactoryProvider() { + } + + public static RecordFactory getRecordFactory(Configuration conf) { + if (conf == null) { + //Assuming the default configuration has the correct factories set. + //Users can specify a particular factory by providing a configuration. + conf = defaultConf; + } + String recordFactoryClassName = conf.get(YarnConfiguration.IPC_RECORD_FACTORY_CLASS, YarnConfiguration.DEFAULT_IPC_RECORD_FACTORY_CLASS); + return (RecordFactory) getFactoryClassInstance(recordFactoryClassName); +// if (recordFactoryClassName == null) { +// String serializer = conf.get(YarnConfiguration.IPC_SERIALIZER_TYPE, YarnConfiguration.DEFAULT_IPC_SERIALIZER_TYPE); +// if (serializer.equals(YarnConfiguration.DEFAULT_IPC_SERIALIZER_TYPE)) { +// return RecordFactoryPBImpl.get(); +// } else { +// throw new YarnException("Unknown serializer: [" + conf.get(YarnConfiguration.IPC_SERIALIZER_TYPE) + "]. Use keys: [" + YarnConfiguration.IPC_RECORD_FACTORY_CLASS + "] to specify Record factory"); +// } +// } else { +// +// } + } + + private static Object getFactoryClassInstance(String factoryClassName) { + try { + Class clazz = Class.forName(factoryClassName); + Method method = clazz.getMethod("get", null); + method.setAccessible(true); + return method.invoke(null, null); + } catch (ClassNotFoundException e) { + throw new YarnException(e); + } catch (NoSuchMethodException e) { + throw new YarnException(e); + } catch (InvocationTargetException e) { + throw new YarnException(e); + } catch (IllegalAccessException e) { + throw new YarnException(e); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/Records.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/Records.java new file mode 100644 index 0000000..60711fd --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/Records.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.util; + +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; + +/** + * Convenient API record utils + */ +public class Records { + // The default record factory + private static final RecordFactory factory = + RecordFactoryProvider.getRecordFactory(null); + + public static T newRecord(Class cls) { + return factory.newRecordInstance(cls); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/YarnException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/YarnException.java deleted file mode 100644 index b12fc81..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/YarnException.java +++ /dev/null @@ -1,29 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn; - -/** Base Yarn Exception. - */ -public class YarnException extends RuntimeException { - public YarnException(Throwable cause) { super(cause); } - public YarnException(String message) { super(message); } - public YarnException(String message, Throwable cause) { - super(message, cause); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java deleted file mode 100644 index f9b017d..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ /dev/null @@ -1,755 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.conf; - -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.util.Arrays; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.ApplicationConstants; - -import com.google.common.base.Joiner; - -public class YarnConfiguration extends Configuration { - - private static final Joiner JOINER = Joiner.on(""); - - private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; - private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; - - static { - Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE); - Configuration.addDefaultResource(YARN_SITE_XML_FILE); - } - - //Configurations - - public static final String YARN_PREFIX = "yarn."; - - /** Delay before deleting resource to ease debugging of NM issues */ - public static final String DEBUG_NM_DELETE_DELAY_SEC = - YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; - - //////////////////////////////// - // IPC Configs - //////////////////////////////// - public static final String IPC_PREFIX = YARN_PREFIX + "ipc."; - - /** Factory to create client IPC classes.*/ - public static final String IPC_CLIENT_FACTORY = - IPC_PREFIX + "client.factory.class"; - - /** Type of serialization to use.*/ - public static final String IPC_SERIALIZER_TYPE = - IPC_PREFIX + "serializer.type"; - public static final String DEFAULT_IPC_SERIALIZER_TYPE = "protocolbuffers"; - - /** Factory to create server IPC classes.*/ - public static final String IPC_SERVER_FACTORY = - IPC_PREFIX + "server.factory.class"; - - /** Factory to create IPC exceptions.*/ - public static final String IPC_EXCEPTION_FACTORY = - IPC_PREFIX + "exception.factory.class"; - - /** Factory to create serializeable records.*/ - public static final String IPC_RECORD_FACTORY = - IPC_PREFIX + "record.factory.class"; - - /** RPC class implementation*/ - public static final String IPC_RPC_IMPL = - IPC_PREFIX + "rpc.class"; - public static final String DEFAULT_IPC_RPC_IMPL = - "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC"; - - //////////////////////////////// - // Resource Manager Configs - //////////////////////////////// - public static final String RM_PREFIX = "yarn.resourcemanager."; - - /** The address of the applications manager interface in the RM.*/ - public static final String RM_ADDRESS = - RM_PREFIX + "address"; - public static final int DEFAULT_RM_PORT = 8032; - public static final String DEFAULT_RM_ADDRESS = - "0.0.0.0:" + DEFAULT_RM_PORT; - - /** The number of threads used to handle applications manager requests.*/ - public static final String RM_CLIENT_THREAD_COUNT = - RM_PREFIX + "client.thread-count"; - public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50; - - /** The Kerberos principal for the resource manager.*/ - public static final String RM_PRINCIPAL = - RM_PREFIX + "principal"; - - /** The address of the scheduler interface.*/ - public static final String RM_SCHEDULER_ADDRESS = - RM_PREFIX + "scheduler.address"; - public static final int DEFAULT_RM_SCHEDULER_PORT = 8030; - public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" + - DEFAULT_RM_SCHEDULER_PORT; - - /** Miniumum request grant-able by the RM scheduler. */ - public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB = - YARN_PREFIX + "scheduler.minimum-allocation-mb"; - public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; - public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = - YARN_PREFIX + "scheduler.minimum-allocation-vcores"; - public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; - - /** Maximum request grant-able by the RM scheduler. */ - public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = - YARN_PREFIX + "scheduler.maximum-allocation-mb"; - public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192; - public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = - YARN_PREFIX + "scheduler.maximum-allocation-vcores"; - public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; - - /** Number of threads to handle scheduler interface.*/ - public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = - RM_PREFIX + "scheduler.client.thread-count"; - public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; - - /** The address of the RM web application.*/ - public static final String RM_WEBAPP_ADDRESS = - RM_PREFIX + "webapp.address"; - - public static final int DEFAULT_RM_WEBAPP_PORT = 8088; - public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" + - DEFAULT_RM_WEBAPP_PORT; - - public static final String RM_RESOURCE_TRACKER_ADDRESS = - RM_PREFIX + "resource-tracker.address"; - public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031; - public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS = - "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT; - - /** The expiry interval for application master reporting.*/ - public static final String RM_AM_EXPIRY_INTERVAL_MS = - YARN_PREFIX + "am.liveness-monitor.expiry-interval-ms"; - public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000; - - /** How long to wait until a node manager is considered dead.*/ - public static final String RM_NM_EXPIRY_INTERVAL_MS = - YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms"; - public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000; - - /** Are acls enabled.*/ - public static final String YARN_ACL_ENABLE = - YARN_PREFIX + "acl.enable"; - public static final boolean DEFAULT_YARN_ACL_ENABLE = false; - - /** ACL of who can be admin of YARN cluster.*/ - public static final String YARN_ADMIN_ACL = - YARN_PREFIX + "admin.acl"; - public static final String DEFAULT_YARN_ADMIN_ACL = "*"; - - /** ACL used in case none is found. Allows nothing. */ - public static final String DEFAULT_YARN_APP_ACL = " "; - - /** The address of the RM admin interface.*/ - public static final String RM_ADMIN_ADDRESS = - RM_PREFIX + "admin.address"; - public static final int DEFAULT_RM_ADMIN_PORT = 8033; - public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" + - DEFAULT_RM_ADMIN_PORT; - - /**Number of threads used to handle RM admin interface.*/ - public static final String RM_ADMIN_CLIENT_THREAD_COUNT = - RM_PREFIX + "admin.client.thread-count"; - public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1; - - /** - * The maximum number of application attempts. - * It's a global setting for all application masters. - */ - public static final String RM_AM_MAX_ATTEMPTS = - RM_PREFIX + "am.max-attempts"; - public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2; - - /** The keytab for the resource manager.*/ - public static final String RM_KEYTAB = - RM_PREFIX + "keytab"; - - /** How long to wait until a container is considered dead.*/ - public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = - RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; - public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000; - - /** Path to file with nodes to include.*/ - public static final String RM_NODES_INCLUDE_FILE_PATH = - RM_PREFIX + "nodes.include-path"; - public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = ""; - - /** Path to file with nodes to exclude.*/ - public static final String RM_NODES_EXCLUDE_FILE_PATH = - RM_PREFIX + "nodes.exclude-path"; - public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = ""; - - /** Number of threads to handle resource tracker calls.*/ - public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = - RM_PREFIX + "resource-tracker.client.thread-count"; - public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50; - - /** The class to use as the resource scheduler.*/ - public static final String RM_SCHEDULER = - RM_PREFIX + "scheduler.class"; - - public static final String DEFAULT_RM_SCHEDULER = - "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"; - - /** RM set next Heartbeat interval for NM */ - public static final String RM_NM_HEARTBEAT_INTERVAL_MS = - RM_PREFIX + "nodemanagers.heartbeat-interval-ms"; - public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000; - - //Delegation token related keys - public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = - RM_PREFIX + "delegation.key.update-interval"; - public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = - 24*60*60*1000; // 1 day - public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY = - RM_PREFIX + "delegation.token.renew-interval"; - public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = - 24*60*60*1000; // 1 day - public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY = - RM_PREFIX + "delegation.token.max-lifetime"; - public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = - 7*24*60*60*1000; // 7 days - - public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled"; - public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false; - - /** The class to use as the persistent store.*/ - public static final String RM_STORE = RM_PREFIX + "store.class"; - - /** URI for FileSystemRMStateStore */ - public static final String FS_RM_STATE_STORE_URI = - RM_PREFIX + "fs.rm-state-store.uri"; - - /** The maximum number of completed applications RM keeps. */ - public static final String RM_MAX_COMPLETED_APPLICATIONS = - RM_PREFIX + "max-completed-applications"; - public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000; - - /** Default application name */ - public static final String DEFAULT_APPLICATION_NAME = "N/A"; - - /** Default queue name */ - public static final String DEFAULT_QUEUE_NAME = "default"; - - /** - * Buckets (in minutes) for the number of apps running in each queue. - */ - public static final String RM_METRICS_RUNTIME_BUCKETS = - RM_PREFIX + "metrics.runtime.buckets"; - - /** - * Default sizes of the runtime metric buckets in minutes. - */ - public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = - "60,300,1440"; - - public static final String RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX - + "application-tokens.master-key-rolling-interval-secs"; - - public static final long DEFAULT_RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = - 24 * 60 * 60; - - public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = - RM_PREFIX + "container-tokens.master-key-rolling-interval-secs"; - - public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = - 24 * 60 * 60; - - //////////////////////////////// - // Node Manager Configs - //////////////////////////////// - - /** Prefix for all node manager configs.*/ - public static final String NM_PREFIX = "yarn.nodemanager."; - - /** Environment variables that will be sent to containers.*/ - public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env"; - public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX"; - - /** Environment variables that containers may override rather than use NodeManager's default.*/ - public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist"; - public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",", - Arrays.asList(ApplicationConstants.Environment.JAVA_HOME.key(), - ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(), - ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(), - ApplicationConstants.Environment.HADOOP_CONF_DIR.key(), - ApplicationConstants.Environment.HADOOP_YARN_HOME.key())); - - /** address of node manager IPC.*/ - public static final String NM_ADDRESS = NM_PREFIX + "address"; - public static final int DEFAULT_NM_PORT = 0; - public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:" - + DEFAULT_NM_PORT; - - /** who will execute(launch) the containers.*/ - public static final String NM_CONTAINER_EXECUTOR = - NM_PREFIX + "container-executor.class"; - - /** - * Adjustment to make to the container os scheduling priority. - * The valid values for this could vary depending on the platform. - * On Linux, higher values mean run the containers at a less - * favorable priority than the NM. - * The value specified is an int. - */ - public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = - NM_PREFIX + "container-executor.os.sched.priority.adjustment"; - public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0; - - /** Number of threads container manager uses.*/ - public static final String NM_CONTAINER_MGR_THREAD_COUNT = - NM_PREFIX + "container-manager.thread-count"; - public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; - - /** Number of threads used in cleanup.*/ - public static final String NM_DELETE_THREAD_COUNT = - NM_PREFIX + "delete.thread-count"; - public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4; - - /** Keytab for NM.*/ - public static final String NM_KEYTAB = NM_PREFIX + "keytab"; - - /**List of directories to store localized files in.*/ - public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs"; - public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir"; - - /** - * Number of files in each localized directories - * Avoid tuning this too low. - */ - public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = - NM_PREFIX + "local-cache.max-files-per-directory"; - public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192; - - /** Address where the localizer IPC is.*/ - public static final String NM_LOCALIZER_ADDRESS = - NM_PREFIX + "localizer.address"; - public static final int DEFAULT_NM_LOCALIZER_PORT = 8040; - public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" + - DEFAULT_NM_LOCALIZER_PORT; - - /** Interval in between cache cleanups.*/ - public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = - NM_PREFIX + "localizer.cache.cleanup.interval-ms"; - public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = - 10 * 60 * 1000; - - /** Target size of localizer cache in MB, per local directory.*/ - public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB = - NM_PREFIX + "localizer.cache.target-size-mb"; - public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024; - - /** Number of threads to handle localization requests.*/ - public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT = - NM_PREFIX + "localizer.client.thread-count"; - public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5; - - /** Number of threads to use for localization fetching.*/ - public static final String NM_LOCALIZER_FETCH_THREAD_COUNT = - NM_PREFIX + "localizer.fetch.thread-count"; - public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4; - - /** Where to store container logs.*/ - public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; - public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; - - /** Interval at which the delayed token removal thread runs */ - public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = - RM_PREFIX + "delayed.delegation-token.removal-interval-ms"; - public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = - 30000l; - - /** Whether to enable log aggregation */ - public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX - + "log-aggregation-enable"; - public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false; - - /** - * How long to wait before deleting aggregated logs, -1 disables. - * Be careful set this too small and you will spam the name node. - */ - public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX - + "log-aggregation.retain-seconds"; - public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; - - /** - * How long to wait between aggregated log retention checks. If set to - * a value <= 0 then the value is computed as one-tenth of the log retention - * setting. Be careful set this too small and you will spam the name node. - */ - public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = - YARN_PREFIX + "log-aggregation.retain-check-interval-seconds"; - public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1; - - /** - * Number of seconds to retain logs on the NodeManager. Only applicable if Log - * aggregation is disabled - */ - public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX - + "log.retain-seconds"; - - /** - * Number of threads used in log cleanup. Only applicable if Log aggregation - * is disabled - */ - public static final String NM_LOG_DELETION_THREADS_COUNT = - NM_PREFIX + "log.deletion-threads-count"; - public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4; - - /** Where to aggregate logs to.*/ - public static final String NM_REMOTE_APP_LOG_DIR = - NM_PREFIX + "remote-app-log-dir"; - public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs"; - - /** - * The remote log dir will be created at - * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId} - */ - public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = - NM_PREFIX + "remote-app-log-dir-suffix"; - public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs"; - - public static final String YARN_LOG_SERVER_URL = - YARN_PREFIX + "log.server.url"; - - public static final String YARN_TRACKING_URL_GENERATOR = - YARN_PREFIX + "tracking.url.generator"; - - /** Amount of memory in GB that can be allocated for containers.*/ - public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb"; - public static final int DEFAULT_NM_PMEM_MB = 8 * 1024; - - /** Specifies whether physical memory check is enabled. */ - public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX - + "pmem-check-enabled"; - public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true; - - /** Specifies whether physical memory check is enabled. */ - public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX - + "vmem-check-enabled"; - public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true; - - /** Conversion ratio for physical memory to virtual memory. */ - public static final String NM_VMEM_PMEM_RATIO = - NM_PREFIX + "vmem-pmem-ratio"; - public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f; - - /** Number of Physical CPU Cores which can be allocated for containers.*/ - public static final String NM_VCORES = NM_PREFIX + "resource.cpu-cores"; - public static final int DEFAULT_NM_VCORES = 8; - - /** Conversion ratio for physical cores to virtual cores. */ - public static final String NM_VCORES_PCORES_RATIO = - NM_PREFIX + "vcores-pcores-ratio"; - public static final float DEFAULT_NM_VCORES_PCORES_RATIO = 2.0f; - - /** NM Webapp address.**/ - public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; - public static final int DEFAULT_NM_WEBAPP_PORT = 8042; - public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" + - DEFAULT_NM_WEBAPP_PORT; - - /** How often to monitor containers.*/ - public final static String NM_CONTAINER_MON_INTERVAL_MS = - NM_PREFIX + "container-monitor.interval-ms"; - public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000; - - /** Class that calculates containers current resource utilization.*/ - public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR = - NM_PREFIX + "container-monitor.resource-calculator.class"; - /** Class that calculates process tree resource utilization.*/ - public static final String NM_CONTAINER_MON_PROCESS_TREE = - NM_PREFIX + "container-monitor.process-tree.class"; - - /** - * Enable/Disable disks' health checker. Default is true. - * An expert level configuration property. - */ - public static final String NM_DISK_HEALTH_CHECK_ENABLE = - NM_PREFIX + "disk-health-checker.enable"; - /** Frequency of running disks' health checker.*/ - public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS = - NM_PREFIX + "disk-health-checker.interval-ms"; - /** By default, disks' health is checked every 2 minutes. */ - public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS = - 2 * 60 * 1000; - - /** - * The minimum fraction of number of disks to be healthy for the nodemanager - * to launch new containers. This applies to nm-local-dirs and nm-log-dirs. - */ - public static final String NM_MIN_HEALTHY_DISKS_FRACTION = - NM_PREFIX + "disk-health-checker.min-healthy-disks"; - /** - * By default, at least 5% of disks are to be healthy to say that the node - * is healthy in terms of disks. - */ - public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION - = 0.25F; - - /** Frequency of running node health script.*/ - public static final String NM_HEALTH_CHECK_INTERVAL_MS = - NM_PREFIX + "health-checker.interval-ms"; - public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000; - - /** Health check script time out period.*/ - public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = - NM_PREFIX + "health-checker.script.timeout-ms"; - public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = - 2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS; - - /** The health check script to run.*/ - public static final String NM_HEALTH_CHECK_SCRIPT_PATH = - NM_PREFIX + "health-checker.script.path"; - - /** The arguments to pass to the health check script.*/ - public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = - NM_PREFIX + "health-checker.script.opts"; - - /** The path to the Linux container executor.*/ - public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = - NM_PREFIX + "linux-container-executor.path"; - - /** - * The UNIX group that the linux-container-executor should run as. - * This is intended to be set as part of container-executor.cfg. - */ - public static final String NM_LINUX_CONTAINER_GROUP = - NM_PREFIX + "linux-container-executor.group"; - - /** The type of resource enforcement to use with the - * linux container executor. - */ - public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER = - NM_PREFIX + "linux-container-executor.resources-handler.class"; - - /** The path the linux container executor should use for cgroups */ - public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY = - NM_PREFIX + "linux-container-executor.cgroups.hierarchy"; - - /** Whether the linux container executor should mount cgroups if not found */ - public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT = - NM_PREFIX + "linux-container-executor.cgroups.mount"; - - /** Where the linux container executor should mount cgroups if not found */ - public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = - NM_PREFIX + "linux-container-executor.cgroups.mount-path"; - - /** T-file compression types used to compress aggregated logs.*/ - public static final String NM_LOG_AGG_COMPRESSION_TYPE = - NM_PREFIX + "log-aggregation.compression-type"; - public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none"; - - /** The kerberos principal for the node manager.*/ - public static final String NM_PRINCIPAL = - NM_PREFIX + "principal"; - - public static final String NM_AUX_SERVICES = - NM_PREFIX + "aux-services"; - - public static final String NM_AUX_SERVICE_FMT = - NM_PREFIX + "aux-services.%s.class"; - - public static final String NM_USER_HOME_DIR = - NM_PREFIX + "user-home-dir"; - - public static final String DEFAULT_NM_USER_HOME_DIR= "/home/"; - - //////////////////////////////// - // Web Proxy Configs - //////////////////////////////// - public static final String PROXY_PREFIX = "yarn.web-proxy."; - - /** The kerberos principal for the proxy.*/ - public static final String PROXY_PRINCIPAL = - PROXY_PREFIX + "principal"; - - /** Keytab for Proxy.*/ - public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab"; - - /** The address for the web proxy.*/ - public static final String PROXY_ADDRESS = - PROXY_PREFIX + "address"; - - /** - * YARN Service Level Authorization - */ - public static final String - YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER = - "security.resourcetracker.protocol.acl"; - public static final String - YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER = - "security.client.resourcemanager.protocol.acl"; - public static final String - YARN_SECURITY_SERVICE_AUTHORIZATION_ADMIN = - "security.admin.protocol.acl"; - public static final String - YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_RESOURCEMANAGER = - "security.applicationmaster.resourcemanager.protocol.acl"; - - public static final String - YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGER = - "security.containermanager.protocol.acl"; - public static final String - YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER = - "security.resourcelocalizer.protocol.acl"; - - /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL - * to a running container */ - public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = - NM_PREFIX + "sleep-delay-before-sigkill.ms"; - public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = - 250; - - /** Max time to wait for a process to come up when trying to cleanup - * container resources */ - public static final String NM_PROCESS_KILL_WAIT_MS = - NM_PREFIX + "process-kill-wait.ms"; - public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS = - 2000; - - /** Max time to wait to establish a connection to RM when NM starts - */ - public static final String RESOURCEMANAGER_CONNECT_WAIT_SECS = - NM_PREFIX + "resourcemanager.connect.wait.secs"; - public static final int DEFAULT_RESOURCEMANAGER_CONNECT_WAIT_SECS = - 15*60; - - /** Time interval between each NM attempt to connect to RM - */ - public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS = - NM_PREFIX + "resourcemanager.connect.retry_interval.secs"; - public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS - = 30; - - /** - * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH - * entries - */ - public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX - + "application.classpath"; - - /** - * Default CLASSPATH for YARN applications. A comma-separated list of - * CLASSPATH entries - */ - public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = { - ApplicationConstants.Environment.HADOOP_CONF_DIR.$(), - ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() - + "/share/hadoop/common/*", - ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() - + "/share/hadoop/common/lib/*", - ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() - + "/share/hadoop/hdfs/*", - ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() - + "/share/hadoop/hdfs/lib/*", - ApplicationConstants.Environment.HADOOP_YARN_HOME.$() - + "/share/hadoop/yarn/*", - ApplicationConstants.Environment.HADOOP_YARN_HOME.$() - + "/share/hadoop/yarn/lib/*" }; - - /** Container temp directory */ - public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; - - public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX - + "is.minicluster"; - - /** Whether to use fixed ports with the minicluster. */ - public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX - + "minicluster.fixed.ports"; - - /** - * Default is false to be able to run tests concurrently without port - * conflicts. - */ - public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false; - - //////////////////////////////// - // Other Configs - //////////////////////////////// - - /** - * The interval of the yarn client's querying application state after - * application submission. The unit is millisecond. - */ - public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = - YARN_PREFIX + "client.app-submission.poll-interval"; - public static final long DEFAULT_YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = - 1000; - - public YarnConfiguration() { - super(); - } - - public YarnConfiguration(Configuration conf) { - super(conf); - if (! (conf instanceof YarnConfiguration)) { - this.reloadConfiguration(); - } - } - - public static String getProxyHostAndPort(Configuration conf) { - String addr = conf.get(PROXY_ADDRESS); - if(addr == null || addr.isEmpty()) { - addr = getRMWebAppHostAndPort(conf); - } - return addr; - } - - public static String getRMWebAppHostAndPort(Configuration conf) { - InetSocketAddress address = conf.getSocketAddr( - YarnConfiguration.RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); - address = NetUtils.getConnectAddress(address); - StringBuffer sb = new StringBuffer(); - InetAddress resolved = address.getAddress(); - if (resolved == null || resolved.isAnyLocalAddress() || - resolved.isLoopbackAddress()) { - String lh = address.getHostName(); - try { - lh = InetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException e) { - //Ignore and fallback. - } - sb.append(lh); - } else { - sb.append(address.getHostName()); - } - sb.append(":").append(address.getPort()); - return sb.toString(); - } - - public static String getRMWebAppURL(Configuration conf) { - return JOINER.join("http://", getRMWebAppHostAndPort(conf)); - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RecordFactory.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RecordFactory.java deleted file mode 100644 index e62d5a6..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RecordFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.factories; - -import org.apache.hadoop.yarn.YarnException; - - -public interface RecordFactory { - public T newRecordInstance(Class clazz) throws YarnException; -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factory/providers/RecordFactoryProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factory/providers/RecordFactoryProvider.java deleted file mode 100644 index 66ed159..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factory/providers/RecordFactoryProvider.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.factory.providers; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.YarnException; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl; - -public class RecordFactoryProvider { - private static Configuration defaultConf; - - static { - defaultConf = new Configuration(); - } - - private RecordFactoryProvider() { - } - - public static RecordFactory getRecordFactory(Configuration conf) { - if (conf == null) { - //Assuming the default configuration has the correct factories set. - //Users can specify a particular factory by providing a configuration. - conf = defaultConf; - } - String recordFactoryClassName = conf.get(YarnConfiguration.IPC_RECORD_FACTORY); - if (recordFactoryClassName == null) { - String serializer = conf.get(YarnConfiguration.IPC_SERIALIZER_TYPE, YarnConfiguration.DEFAULT_IPC_SERIALIZER_TYPE); - if (serializer.equals(YarnConfiguration.DEFAULT_IPC_SERIALIZER_TYPE)) { - return RecordFactoryPBImpl.get(); - } else { - throw new YarnException("Unknown serializer: [" + conf.get(YarnConfiguration.IPC_SERIALIZER_TYPE) + "]. Use keys: [" + YarnConfiguration.IPC_RECORD_FACTORY + "] to specify Record factory"); - } - } else { - return (RecordFactory) getFactoryClassInstance(recordFactoryClassName); - } - } - - private static Object getFactoryClassInstance(String factoryClassName) { - try { - Class clazz = Class.forName(factoryClassName); - Method method = clazz.getMethod("get", null); - method.setAccessible(true); - return method.invoke(null, null); - } catch (ClassNotFoundException e) { - throw new YarnException(e); - } catch (NoSuchMethodException e) { - throw new YarnException(e); - } catch (InvocationTargetException e) { - throw new YarnException(e); - } catch (IllegalAccessException e) { - throw new YarnException(e); - } - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Records.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Records.java deleted file mode 100644 index 60711fd..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Records.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.util; - -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; - -/** - * Convenient API record utils - */ -public class Records { - // The default record factory - private static final RecordFactory factory = - RecordFactoryProvider.getRecordFactory(null); - - public static T newRecord(Class cls) { - return factory.newRecordInstance(cls); - } -}