diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 089b88cdb0..5b91785abc 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -627,6 +627,14 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "internal usage only, used only in test mode. If set false, the operation logs, and the " + "operation log directory will not be removed, so they can be found after the test runs."), + HIVE_TEST_LOAD_ENABLED("hive.test.load.enabled", false, + "Enables a CPU load testing"), + HIVE_TEST_LOAD_HOSTNAME("hive.test.load.hostnames", "", + "Specify host names for load testing. (e.g., \"host1,host2,host3\")"), + HIVE_TEST_LOAD_UTILIZATION("hive.test.load.utilization", 0.2f, + "Specify processor load utilization for between 0.0 (not loaded on all threads) and 1.0 " + + "(fully loaded on all threads)."), + HIVE_IN_TEZ_TEST("hive.in.tez.test", false, "internal use only, true when in testing tez", true), HIVE_MAPJOIN_TESTING_NO_HASH_TABLE_LOAD("hive.mapjoin.testing.no.hash.table.load", false, "internal use only, true when in testing map join", diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index bafa0294df..5c55d64ad8 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -100,6 +100,7 @@ private final AMReporter amReporter; private final LlapRegistryService registry; private final LlapWebServices webServices; + private final LlapLoadGeneratorService llapLoadGeneratorService; private final AtomicLong numSubmissions = new AtomicLong(0); private final JvmPauseMonitor pauseMonitor; private final ObjectName llapDaemonInfoBean; @@ -323,6 +324,13 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor this.webServices = new LlapWebServices(webPort, this, registry); addIfService(webServices); } + + if (!HiveConf.getBoolVar(daemonConf, ConfVars.HIVE_TEST_LOAD_ENABLED)) { + this.llapLoadGeneratorService = null; + } else { + this.llapLoadGeneratorService = new LlapLoadGeneratorService(); + addIfService(llapLoadGeneratorService); + } // Bring up the server only after all other components have started. addIfService(server); // AMReporter after the server so that it gets the correct address. It knows how to deal with diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapLoadGeneratorService.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapLoadGeneratorService.java new file mode 100644 index 0000000000..e0955d3276 --- /dev/null +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapLoadGeneratorService.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.daemon.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.service.AbstractService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Random; + +/** + * Extra load generator service for LLAP. + */ +public class LlapLoadGeneratorService extends AbstractService { + private static final Logger LOG = LoggerFactory.getLogger(LlapLoadGeneratorService.class); + private static final long INTERVAL = 10; + private float threshold; + private String[] hostNames; + private int processors; + private Thread[] threads; + + public LlapLoadGeneratorService() { + super("LlapLoadGeneratorService"); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + super.serviceInit(conf); + threshold = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVE_TEST_LOAD_UTILIZATION); + hostNames = HiveConf.getTrimmedStringsVar(conf, HiveConf.ConfVars.HIVE_TEST_LOAD_HOSTNAME); + LOG.info("LlapLoadGeneratorService init with {} {}", threshold, hostNames); + } + + protected void serviceStart() throws UnknownHostException { + String localHostName = InetAddress.getLocalHost().getHostName(); + LOG.debug("Local hostname is: {}", localHostName); + for (String hostName : hostNames) { + if (hostName.equalsIgnoreCase(localHostName)) { + LOG.debug("Starting load generator process on: {}", localHostName); + processors = Runtime.getRuntime().availableProcessors(); + threads = new Thread[processors]; + Random random = new Random(); + for (int i = 0; i < processors; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + while (!Thread.interrupted()) { + if (random.nextFloat() <= threshold) { + // Keep it busy + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < INTERVAL) { + // active loop, do nothing + } + } else { + // Keep it idle + try { + Thread.sleep(INTERVAL); + } catch (InterruptedException e) { + // Ignore + } + } + } + } + }); + threads[i].start(); + } + } + } + } + + @Override + protected void serviceStop() throws Exception { + for (int i = 0; i < processors; i++) { + threads[i].interrupt(); + } + } +} diff --git llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapLoadGeneratorService.java llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapLoadGeneratorService.java new file mode 100644 index 0000000000..5f2a01deb8 --- /dev/null +++ llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapLoadGeneratorService.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.daemon.impl; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * Test to make sure that the LLAP nodes are able to start with the load generator. + */ +public class TestLlapLoadGeneratorService { + @Test + public void testLoadGenerator() throws InterruptedException, UnknownHostException { + LlapLoadGeneratorService service = new LlapLoadGeneratorService(); + + HiveConf conf = new HiveConf(); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_TEST_LOAD_ENABLED, true); + HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_TEST_LOAD_HOSTNAME, + InetAddress.getLocalHost().getHostName() + ",???"); + HiveConf.setFloatVar(conf, HiveConf.ConfVars.HIVE_TEST_LOAD_UTILIZATION, 0.2f); + + service.init(conf); + service.start(); + Thread.sleep(10000); + service.stop(); + } +}