diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java new file mode 100644 index 0000000..1b1ad85 --- /dev/null +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.llap.configuration; + +import org.apache.hadoop.conf.Configuration; + +public class LlapConfiguration extends Configuration { + + public LlapConfiguration(Configuration conf) { + super(conf); + addResource(LLAP_DAEMON_SITE); + } + + public LlapConfiguration() { + super(false); + addResource(LLAP_DAEMON_SITE); + } + + + public static final String LLAP_DAEMON_PREFIX = "llap.daemon."; + private static final String LLAP_DAEMON_SITE = "llap-daemon-site.xml"; + + + + public static final String LLAP_DAEMON_RPC_NUM_HANDLERS = LLAP_DAEMON_PREFIX + "rpc.num.handlers"; + public static final int LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT = 5; + + public static final String LLAP_DAEMON_WORK_DIRS = LLAP_DAEMON_PREFIX + "work.dirs"; + + public static final String LLAP_DAEMON_YARN_SHUFFLE_PORT = LLAP_DAEMON_PREFIX + "yarn.shuffle.port"; + public static final int LLAP_DAEMON_YARN_SHUFFLE_PORT_DEFAULT = 15551; + + public static final String LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED = LLAP_DAEMON_PREFIX + "shuffle.dir-watcher.enabled"; + public static final boolean LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED_DEFAULT = false; + + + // Section for configs used in AM and executors + public static final String LLAP_DAEMON_NUM_EXECUTORS = LLAP_DAEMON_PREFIX + "num.executors"; + public static final int LLAP_DAEMON_NUM_EXECUTORS_DEFAULT = 4; + + public static final String LLAP_DAEMON_RPC_PORT = LLAP_DAEMON_PREFIX + "rpc.port"; + public static final int LLAP_DAEMON_RPC_PORT_DEFAULT = 15001; + + public static final String LLAP_DAEMON_MEMORY_PER_INSTANCE_MB = LLAP_DAEMON_PREFIX + "memory.per.instance.mb"; + public static final int LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT = 4096; + + public static final String LLAP_DAEMON_VCPUS_PER_INSTANCE = LLAP_DAEMON_PREFIX + "vcpus.per.instance"; + public static final int LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT = 4; + + + // Section for configs used in the AM // + public static final String LLAP_DAEMON_SERVICE_HOSTS = LLAP_DAEMON_PREFIX + "service.hosts"; + + public static final String LLAP_DAEMON_COMMUNICATOR_NUM_THREADS = LLAP_DAEMON_PREFIX + "communicator.num.threads"; + public static final int LLAP_DAEMON_COMMUNICATOR_NUM_THREADS_DEFAULT = 5; + + /** + * Time after which a previously disabled node will be re-enabled for scheduling. This may be + * modified by an exponential back-off if failures persist + */ + public static final String LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS = + LLAP_DAEMON_PREFIX + "task.scheduler.node.re-enable.timeout.ms"; + public static final long LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS_DEFAULT = 2000l; + + +} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index ca22a21..6a5f3fc 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hive.common.CompressionUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.cli.LlapOptionsProcessor.LlapOptions; -import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; +import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.io.api.impl.LlapInputFormat; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.HiveInputFormat; @@ -121,7 +121,7 @@ private int run(String[] args) throws Exception { if (options.getName() != null) { // update service registry configs - caveat: this has nothing to do with the actual settings as read by the AM // if needed, use --hiveconf llap.daemon.service.hosts=@llap0 to dynamically switch between instances - conf.set(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS, "@" + options.getName()); + conf.set(LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS, "@" + options.getName()); } URL logger = conf.getResource("llap-daemon-log4j.properties"); @@ -188,17 +188,17 @@ private int run(String[] args) throws Exception { configs.put(HiveConf.ConfVars.LLAP_ORC_CACHE_ALLOCATE_DIRECT.varname, HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_ORC_CACHE_ALLOCATE_DIRECT)); - configs.put(LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, conf.getInt( - LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, - LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT)); + configs.put(LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, conf.getInt( + LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, + LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT)); - configs.put(LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE, conf.getInt( - LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE, - LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT)); + configs.put(LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE, conf.getInt( + LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE, + LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT)); - configs.put(LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS, conf.getInt( - LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS, - LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT)); + configs.put(LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS, conf.getInt( + LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS, + LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT)); configs.put(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1)); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java deleted file mode 100644 index a820e26..0000000 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.llap.daemon; - -import org.apache.hadoop.conf.Configuration; - -public class LlapDaemonConfiguration extends Configuration { - - public LlapDaemonConfiguration(Configuration conf) { - super(conf); - addResource(LLAP_DAEMON_SITE); - } - - public LlapDaemonConfiguration() { - super(false); - addResource(LLAP_DAEMON_SITE); - } - - - public static final String LLAP_DAEMON_PREFIX = "llap.daemon."; - private static final String LLAP_DAEMON_SITE = "llap-daemon-site.xml"; - - - - public static final String LLAP_DAEMON_RPC_NUM_HANDLERS = LLAP_DAEMON_PREFIX + "rpc.num.handlers"; - public static final int LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT = 5; - - public static final String LLAP_DAEMON_WORK_DIRS = LLAP_DAEMON_PREFIX + "work.dirs"; - - public static final String LLAP_DAEMON_YARN_SHUFFLE_PORT = LLAP_DAEMON_PREFIX + "yarn.shuffle.port"; - public static final int LLAP_DAEMON_YARN_SHUFFLE_PORT_DEFAULT = 15551; - - public static final String LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED = LLAP_DAEMON_PREFIX + "shuffle.dir-watcher.enabled"; - public static final boolean LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED_DEFAULT = false; - - - // Section for configs used in AM and executors - public static final String LLAP_DAEMON_NUM_EXECUTORS = LLAP_DAEMON_PREFIX + "num.executors"; - public static final int LLAP_DAEMON_NUM_EXECUTORS_DEFAULT = 4; - - public static final String LLAP_DAEMON_RPC_PORT = LLAP_DAEMON_PREFIX + "rpc.port"; - public static final int LLAP_DAEMON_RPC_PORT_DEFAULT = 15001; - - public static final String LLAP_DAEMON_MEMORY_PER_INSTANCE_MB = LLAP_DAEMON_PREFIX + "memory.per.instance.mb"; - public static final int LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT = 4096; - - public static final String LLAP_DAEMON_VCPUS_PER_INSTANCE = LLAP_DAEMON_PREFIX + "vcpus.per.instance"; - public static final int LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT = 4; - - - // Section for configs used in the AM // - public static final String LLAP_DAEMON_SERVICE_HOSTS = LLAP_DAEMON_PREFIX + "service.hosts"; - - public static final String LLAP_DAEMON_COMMUNICATOR_NUM_THREADS = LLAP_DAEMON_PREFIX + "communicator.num.threads"; - public static final int LLAP_DAEMON_COMMUNICATOR_NUM_THREADS_DEFAULT = 5; - - /** - * Time after which a previously disabled node will be re-enabled for scheduling. This may be - * modified by an exponential back-off if failures persist - */ - public static final String LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS = - LLAP_DAEMON_PREFIX + "task.scheduler.node.re-enable.timeout.ms"; - public static final long LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS_DEFAULT = 2000l; - - -} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index a5f7c0e..56d7313 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -24,8 +24,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.ContainerRunner; -import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.registry.impl.LlapRegistryService; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos; import org.apache.hadoop.hive.llap.io.api.LlapIoProxy; @@ -110,12 +110,12 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor this.shuffleHandlerConf.set(ShuffleHandler.SHUFFLE_HANDLER_LOCAL_DIRS, StringUtils.arrayToString(localDirs)); this.shuffleHandlerConf.setBoolean(ShuffleHandler.SHUFFLE_DIR_WATCHER_ENABLED, daemonConf - .getBoolean(LlapDaemonConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED, - LlapDaemonConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED_DEFAULT)); + .getBoolean(LlapConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED, + LlapConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED_DEFAULT)); // Less frequently set parameter, not passing in as a param. - int numHandlers = daemonConf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS, - LlapDaemonConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT); + int numHandlers = daemonConf.getInt(LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS, + LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT); this.server = new LlapDaemonProtocolServerImpl(numHandlers, this, address, rpcPort); // Initialize the metric system @@ -201,18 +201,18 @@ public static void main(String[] args) throws Exception { try { // Cache settings will need to be setup in llap-daemon-site.xml - since the daemons don't read hive-site.xml // Ideally, these properties should be part of LlapDameonConf rather than HiveConf - LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration(); - int numExecutors = daemonConf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS, - LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT); + LlapConfiguration daemonConf = new LlapConfiguration(); + int numExecutors = daemonConf.getInt(LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS, + LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT); String[] localDirs = - daemonConf.getTrimmedStrings(LlapDaemonConfiguration.LLAP_DAEMON_WORK_DIRS); - int rpcPort = daemonConf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_RPC_PORT, - LlapDaemonConfiguration.LLAP_DAEMON_RPC_PORT_DEFAULT); + daemonConf.getTrimmedStrings(LlapConfiguration.LLAP_DAEMON_WORK_DIRS); + int rpcPort = daemonConf.getInt(LlapConfiguration.LLAP_DAEMON_RPC_PORT, + LlapConfiguration.LLAP_DAEMON_RPC_PORT_DEFAULT); int shufflePort = daemonConf .getInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, ShuffleHandler.DEFAULT_SHUFFLE_PORT); long executorMemoryBytes = daemonConf - .getInt(LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, - LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT) * 1024l * 1024l; + .getInt(LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, + LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT) * 1024l * 1024l; long cacheMemoryBytes = HiveConf.getLongVar(daemonConf, HiveConf.ConfVars.LLAP_ORC_CACHE_MAX_SIZE); boolean llapIoEnabled = HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.LLAP_IO_ENABLED); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java index 33cd7eb..67596fe 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java @@ -8,7 +8,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; +import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; @@ -56,7 +56,7 @@ public LlapRegistryService() { @Override public void serviceInit(Configuration conf) { - String registryId = conf.getTrimmed(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS); + String registryId = conf.getTrimmed(LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS); if (registryId.startsWith("@")) { LOG.info("Llap Registry is enabled with registryid: " + registryId); this.conf = new Configuration(conf); @@ -86,16 +86,16 @@ public void serviceStop() throws Exception { public Endpoint getRpcEndpoint() { final int rpcPort = - conf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_RPC_PORT, - LlapDaemonConfiguration.LLAP_DAEMON_RPC_PORT_DEFAULT); + conf.getInt(LlapConfiguration.LLAP_DAEMON_RPC_PORT, + LlapConfiguration.LLAP_DAEMON_RPC_PORT_DEFAULT); return RegistryTypeUtils.ipcEndpoint("llap", new InetSocketAddress(hostname, rpcPort)); } public Endpoint getShuffleEndpoint() { final int shufflePort = - conf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_YARN_SHUFFLE_PORT, - LlapDaemonConfiguration.LLAP_DAEMON_YARN_SHUFFLE_PORT_DEFAULT); + conf.getInt(LlapConfiguration.LLAP_DAEMON_YARN_SHUFFLE_PORT, + LlapConfiguration.LLAP_DAEMON_YARN_SHUFFLE_PORT_DEFAULT); // HTTP today, but might not be return RegistryTypeUtils.inetAddrEndpoint("shuffle", ProtocolTypes.PROTOCOL_TCP, hostname, shufflePort); @@ -114,7 +114,7 @@ public void registerWorker() throws IOException { srv.addInternalEndpoint(getShuffleEndpoint()); for (Map.Entry kv : this.conf) { - if (kv.getKey().startsWith(LlapDaemonConfiguration.LLAP_DAEMON_PREFIX)) { + if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX)) { // TODO: read this somewhere useful, like the allocator srv.set(kv.getKey(), kv.getValue()); } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java index e2f6325..c376def 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java @@ -26,7 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; +import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto; import org.apache.hadoop.io.DataOutputBuffer; @@ -72,8 +72,8 @@ public LlapTaskCommunicator( @Override public void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); - int numThreads = conf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_COMMUNICATOR_NUM_THREADS, - LlapDaemonConfiguration.LLAP_DAEMON_COMMUNICATOR_NUM_THREADS_DEFAULT); + int numThreads = conf.getInt(LlapConfiguration.LLAP_DAEMON_COMMUNICATOR_NUM_THREADS, + LlapConfiguration.LLAP_DAEMON_COMMUNICATOR_NUM_THREADS_DEFAULT); this.communicator = new TaskCommunicator(numThreads); this.communicator.init(conf); } diff --git a/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java b/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java index b944101..d954dbf 100644 --- a/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java +++ b/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java @@ -63,7 +63,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; +import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.registry.impl.LlapRegistryService; import org.apache.hadoop.yarn.util.Clock; import org.apache.tez.dag.api.TaskAttemptEndReason; @@ -168,31 +168,31 @@ public LlapTaskSchedulerService(TaskSchedulerAppCallback appClient, AppContext a this.clock = appContext.getClock(); this.containerFactory = new ContainerFactory(appContext, customAppIdIdentifier); this.memoryPerInstance = conf - .getInt(LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, - LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT); + .getInt(LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, + LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT); this.coresPerInstance = conf - .getInt(LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE, - LlapDaemonConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT); - this.executorsPerInstance = conf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS, - LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT); + .getInt(LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE, + LlapConfiguration.LLAP_DAEMON_VCPUS_PER_INSTANCE_DEFAULT); + this.executorsPerInstance = conf.getInt(LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS, + LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT); this.nodeReEnableTimeout = conf.getLong( - LlapDaemonConfiguration.LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS, - LlapDaemonConfiguration.LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS_DEFAULT); + LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS, + LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS_DEFAULT); int memoryPerExecutor = (int) (memoryPerInstance / (float) executorsPerInstance); int coresPerExecutor = (int) (coresPerInstance / (float) executorsPerInstance); this.resourcePerExecutor = Resource.newInstance(memoryPerExecutor, coresPerExecutor); - String instanceId = conf.getTrimmed(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS); + String instanceId = conf.getTrimmed(LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS); Preconditions.checkNotNull(instanceId, - LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS + " must be defined"); + LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS + " must be defined"); if (!instanceId.startsWith("@")) { // Manual setup. Not via the service registry initFromRegistry = false; - String[] hosts = conf.getTrimmedStrings(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS); + String[] hosts = conf.getTrimmedStrings(LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS); Preconditions.checkState(hosts != null && hosts.length != 0, - LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS + "must be defined"); + LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS + "must be defined"); for (String host : hosts) { NodeInfo nodeInfo = new NodeInfo(host, BACKOFF_FACTOR, clock); activeHosts.put(host, nodeInfo); @@ -203,8 +203,8 @@ public LlapTaskSchedulerService(TaskSchedulerAppCallback appClient, AppContext a initFromRegistry = true; } - this.containerPort = conf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_RPC_PORT, - LlapDaemonConfiguration.LLAP_DAEMON_RPC_PORT_DEFAULT); + this.containerPort = conf.getInt(LlapConfiguration.LLAP_DAEMON_RPC_PORT, + LlapConfiguration.LLAP_DAEMON_RPC_PORT_DEFAULT); ExecutorService executorService = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapScheduler").build()); executor = MoreExecutors.listeningDecorator(executorService); diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java index 7484424..9fd4ba6 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.Service; @@ -127,16 +128,16 @@ public void serviceInit(Configuration conf) { public void serviceStart() { llapDaemon.start(); - clusterSpecificConfiguration.set(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS, + clusterSpecificConfiguration.set(LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS, getServiceAddress().getHostName()); - clusterSpecificConfiguration.setInt(LlapDaemonConfiguration.LLAP_DAEMON_RPC_PORT, + clusterSpecificConfiguration.setInt(LlapConfiguration.LLAP_DAEMON_RPC_PORT, getServiceAddress().getPort()); clusterSpecificConfiguration.setInt( - LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS, + LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS, numExecutorsPerService); clusterSpecificConfiguration.setLong( - LlapDaemonConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, execBytesPerService); + LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB, execBytesPerService); } @Override diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java index 9e09770..b2ac632 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java @@ -21,8 +21,8 @@ import com.google.protobuf.ServiceException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.hive.llap.daemon.ContainerRunner; -import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto; import org.junit.Test; @@ -32,11 +32,11 @@ @Test(timeout = 10000) public void test() throws ServiceException { - LlapDaemonConfiguration daemonConf = new LlapDaemonConfiguration(); - int rpcPort = daemonConf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_RPC_PORT, - LlapDaemonConfiguration.LLAP_DAEMON_RPC_PORT_DEFAULT); - int numHandlers = daemonConf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS, - LlapDaemonConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT); + LlapConfiguration daemonConf = new LlapConfiguration(); + int rpcPort = daemonConf.getInt(LlapConfiguration.LLAP_DAEMON_RPC_PORT, + LlapConfiguration.LLAP_DAEMON_RPC_PORT_DEFAULT); + int numHandlers = daemonConf.getInt(LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS, + LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT); LlapDaemonProtocolServerImpl server = new LlapDaemonProtocolServerImpl(numHandlers, mock(ContainerRunner.class), new AtomicReference(), rpcPort); diff --git a/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java b/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java index 915a6d8..efd4c50 100644 --- a/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java +++ b/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java @@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; +import org.apache.hadoop.hive.llap.configuration.LlapConfiguration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -211,9 +211,9 @@ public void testNodeReEnabled() throws InterruptedException { TestTaskSchedulerServiceWrapper(long disableTimeoutMillis) { conf = new Configuration(); - conf.setStrings(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_HOSTS, HOST1, HOST2, HOST3); - conf.setInt(LlapDaemonConfiguration.LLAP_DAEMON_NUM_EXECUTORS, 4); - conf.setLong(LlapDaemonConfiguration.LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS, disableTimeoutMillis); + conf.setStrings(LlapConfiguration.LLAP_DAEMON_SERVICE_HOSTS, HOST1, HOST2, HOST3); + conf.setInt(LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS, 4); + conf.setLong(LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_NODE_REENABLE_TIMEOUT_MILLIS, disableTimeoutMillis); doReturn(clock).when(mockAppContext).getClock(); doReturn(appAttemptId).when(mockAppContext).getApplicationAttemptId();