diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index da171b1..31e8706 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1779,7 +1779,13 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVE_SSL_PROTOCOL_BLACKLIST("hive.ssl.protocol.blacklist", "SSLv2,SSLv3", "SSL Versions to disable for all Hive Servers"), - // HiveServer2 specific configs + // HiveServer2 specific configs + HIVE_SERVER2_SERVICE_CLASSES("hive.server2.service.classes", "", + "Comma separated class names implementing org.apache.hive.service.Service, which is currently in evolving state." + + "But for backward compatibility, it's recommended to extend org.apache.hive.service.AbstractService " + + "rather than implementing it directly." + + "These services will be managed in HiveServer2 with the same lifecycle of it (start/stop)."), + HIVE_SERVER2_MAX_START_ATTEMPTS("hive.server2.max.start.attempts", 30L, new RangeValidator(0L, null), "Number of times HiveServer2 will attempt to start before exiting, sleeping 60 seconds " + "between retries. \n The default of 30 will keep trying for 30 minutes."), @@ -2003,7 +2009,8 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "(hive.server2.session.check.interval) are enabled."), HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list", - "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role", + "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role," + + "hive.server2.service.classes", "Comma separated list of configuration options which are immutable at runtime"), // If this is set all move tasks at the end of a multi-insert query will only begin once all diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatInputFormat.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatInputFormat.java index 22e7091..0ab6db1 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatInputFormat.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatInputFormat.java @@ -23,9 +23,9 @@ import java.util.Properties; import com.google.common.base.Preconditions; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.mapreduce.Job; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.common.HCatUtil; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index ca86301..b94dfa5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -90,6 +90,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.ContentSummary; @@ -2163,6 +2164,43 @@ public static String getResourceFiles(Configuration conf, SessionState.ResourceT } /** + * Returns the instances specified in a configuration variable as a list + * in the order they were specified in the configuration variable. + * + * @param conf Configuration object + * @param confVar The configuration variable specifying a comma separated list + * of the class names + * @param clazz The super type of the instances + * @return A list of the instances cast as the type specified in clazz, + * in the order they are listed in the value of hookConfVar + * @throws ClassNotFoundException + * @throws IllegalAccessException + * @throws InstantiationException + */ + public static List getInstances(HiveConf conf, ConfVars confVar, Class clazz) + throws InstantiationException, IllegalAccessException, ClassNotFoundException { + String var = conf.getVar(confVar); + if (var == null) { + return Collections.emptyList(); + } + var = var.trim(); + if (var.isEmpty()) { + return Collections.emptyList(); + } + ClassLoader loader = getSessionSpecifiedClassLoader(); + List instances = new ArrayList(); + for (String className : var.split(",")) { + T instance = Class.forName(className.trim(), true, loader).asSubclass(clazz).newInstance(); + if (instance instanceof Configurable) { + ((Configurable) instance).setConf(conf); + } + instances.add(instance); + } + + return instances; + } + + /** * get session specified class loader and get current class loader if fall * * @return diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java index 2f0bd88..965c13c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java @@ -18,15 +18,14 @@ package org.apache.hadoop.hive.ql.hooks; -import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.Utilities; public class HookUtils { + /** * Returns the hooks specified in a configuration variable. The hooks are returned * in a list in the order they were specified in the configuration variable. @@ -44,25 +43,7 @@ public static List getHooks(HiveConf conf, ConfVars hookConfVar, Class clazz) throws InstantiationException, IllegalAccessException, ClassNotFoundException { - String csHooks = conf.getVar(hookConfVar); - List hooks = new ArrayList(); - if (csHooks == null) { - return hooks; - } - - csHooks = csHooks.trim(); - if (csHooks.equals("")) { - return hooks; - } - - String[] hookClasses = csHooks.split(","); - for (String hookClass : hookClasses) { - T hook = (T) Class.forName(hookClass.trim(), true, - Utilities.getSessionSpecifiedClassLoader()).newInstance(); - hooks.add(hook); - } - - return hooks; + return Utilities.getInstances(conf, hookConfVar, clazz); } public static String redactLogString(HiveConf conf, String logString) diff --git a/service/src/java/org/apache/hive/service/AbstractService.java b/service/src/java/org/apache/hive/service/AbstractService.java index c2a2b2d..44baf18 100644 --- a/service/src/java/org/apache/hive/service/AbstractService.java +++ b/service/src/java/org/apache/hive/service/AbstractService.java @@ -23,12 +23,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.conf.HiveConf; /** * AbstractService. * */ +@InterfaceAudience.Public public abstract class AbstractService implements Service { private static final Log LOG = LogFactory.getLog(AbstractService.class); @@ -125,12 +127,22 @@ public synchronized void stop() { LOG.info("Service:" + getName() + " is stopped."); } - @Override + /** + * Register an instance of the service state change events. + * + * @param listener + * a new listener + */ public synchronized void register(ServiceStateChangeListener l) { listeners.add(l); } - @Override + /** + * Unregister a previously instance of the service state change events. + * + * @param listener + * the listener to unregister. + */ public synchronized void unregister(ServiceStateChangeListener l) { listeners.remove(l); } @@ -140,12 +152,24 @@ public String getName() { return name; } - @Override + /** + * Get the configuration of this service. + * This is normally not a clone and may be manipulated, though there are no + * guarantees as to what the consequences of such actions may be + * + * @return the current configuration, unless a specific implementation chooses + * otherwise. + */ public synchronized HiveConf getHiveConf() { return hiveConf; } - @Override + /** + * Get the service start time + * + * @return the start time of the service. This will be zero if the service + * has not yet been started. + */ public long getStartTime() { return startTime; } @@ -174,10 +198,11 @@ private void ensureCurrentState(STATE currentState) { * new service state */ private void changeState(STATE newState) { + STATE prev = state; state = newState; // notify listeners for (ServiceStateChangeListener l : listeners) { - l.stateChanged(this); + l.stateChanged(prev, this); } } diff --git a/service/src/java/org/apache/hive/service/BreakableService.java b/service/src/java/org/apache/hive/service/BreakableService.java index 9c44beb..1180e8f 100644 --- a/service/src/java/org/apache/hive/service/BreakableService.java +++ b/service/src/java/org/apache/hive/service/BreakableService.java @@ -18,8 +18,8 @@ package org.apache.hive.service; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hive.service.Service.STATE; /** * This is a service that can be configured to break on any of the lifecycle @@ -31,6 +31,7 @@ * before the superclass state methods are invoked. * */ +@InterfaceAudience.Private public class BreakableService extends AbstractService { private boolean failOnInit; private boolean failOnStart; diff --git a/service/src/java/org/apache/hive/service/CompositeService.java b/service/src/java/org/apache/hive/service/CompositeService.java index 8979118..3e5fbeb 100644 --- a/service/src/java/org/apache/hive/service/CompositeService.java +++ b/service/src/java/org/apache/hive/service/CompositeService.java @@ -25,12 +25,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.conf.HiveConf; /** * CompositeService. * */ +@InterfaceAudience.Private public class CompositeService extends AbstractService { private static final Log LOG = LogFactory.getLog(CompositeService.class); diff --git a/service/src/java/org/apache/hive/service/FilterService.java b/service/src/java/org/apache/hive/service/FilterService.java index 5a50874..9ff48f3 100644 --- a/service/src/java/org/apache/hive/service/FilterService.java +++ b/service/src/java/org/apache/hive/service/FilterService.java @@ -18,12 +18,14 @@ package org.apache.hive.service; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.conf.HiveConf; /** * FilterService. * */ +@InterfaceAudience.Private public class FilterService implements Service { @@ -51,33 +53,12 @@ public void stop() { @Override - public void register(ServiceStateChangeListener listener) { - service.register(listener); - } - - @Override - public void unregister(ServiceStateChangeListener listener) { - service.unregister(listener); - } - - @Override public String getName() { return service.getName(); } @Override - public HiveConf getHiveConf() { - return service.getHiveConf(); - } - - @Override public STATE getServiceState() { return service.getServiceState(); } - - @Override - public long getStartTime() { - return startTime; - } - } diff --git a/service/src/java/org/apache/hive/service/Service.java b/service/src/java/org/apache/hive/service/Service.java index 2111837..05d8c94 100644 --- a/service/src/java/org/apache/hive/service/Service.java +++ b/service/src/java/org/apache/hive/service/Service.java @@ -18,12 +18,17 @@ package org.apache.hive.service; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.conf.HiveConf; /** * Service. * */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +@InterfaceStability.Unstable public interface Service { /** @@ -54,7 +59,6 @@ */ void init(HiveConf conf); - /** * Start the service. * @@ -72,22 +76,6 @@ void stop(); /** - * Register an instance of the service state change events. - * - * @param listener - * a new listener - */ - void register(ServiceStateChangeListener listener); - - /** - * Unregister a previously instance of the service state change events. - * - * @param listener - * the listener to unregister. - */ - void unregister(ServiceStateChangeListener listener); - - /** * Get the name of this service. * * @return the service name @@ -95,28 +83,9 @@ String getName(); /** - * Get the configuration of this service. - * This is normally not a clone and may be manipulated, though there are no - * guarantees as to what the consequences of such actions may be - * - * @return the current configuration, unless a specific implementation chooses - * otherwise. - */ - HiveConf getHiveConf(); - - /** * Get the current service state * * @return the state of the service */ STATE getServiceState(); - - /** - * Get the service start time - * - * @return the start time of the service. This will be zero if the service - * has not yet been started. - */ - long getStartTime(); - } diff --git a/service/src/java/org/apache/hive/service/ServiceOperations.java b/service/src/java/org/apache/hive/service/ServiceOperations.java index 8946219..6ba04fb 100644 --- a/service/src/java/org/apache/hive/service/ServiceOperations.java +++ b/service/src/java/org/apache/hive/service/ServiceOperations.java @@ -20,12 +20,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.conf.HiveConf; /** * ServiceOperations. * */ +@InterfaceAudience.Private public final class ServiceOperations { private static final Log LOG = LogFactory.getLog(AbstractService.class); diff --git a/service/src/java/org/apache/hive/service/ServiceStateChangeListener.java b/service/src/java/org/apache/hive/service/ServiceStateChangeListener.java index 16ad9a9..d5c7298 100644 --- a/service/src/java/org/apache/hive/service/ServiceStateChangeListener.java +++ b/service/src/java/org/apache/hive/service/ServiceStateChangeListener.java @@ -18,10 +18,13 @@ package org.apache.hive.service; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; + /** * ServiceStateChangeListener. * */ +@InterfaceAudience.Private public interface ServiceStateChangeListener { /** @@ -29,7 +32,7 @@ * have changed state before this callback is invoked. * * This operation is invoked on the thread that initiated the state change, - * while the service itself in in a sychronized section. + * while the service itself in in a synchronized section. *
    *
  1. Any long-lived operation here will prevent the service state * change from completing in a timely manner.
  2. @@ -39,8 +42,9 @@ *
* * + * @param prev * @param service the service that has changed. */ - void stateChanged(Service service); + void stateChanged(Service.STATE prev, Service service); } diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 4a4be97..8178951 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; import org.apache.hadoop.hive.shims.Utils; @@ -57,6 +58,7 @@ import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.common.util.HiveVersionInfo; import org.apache.hive.service.CompositeService; +import org.apache.hive.service.Service; import org.apache.hive.service.cli.CLIService; import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIService; @@ -99,6 +101,11 @@ public synchronized void init(HiveConf hiveConf) { thriftCLIService = new ThriftBinaryCLIService(cliService); } addService(thriftCLIService); + + for (Service service : getServices(hiveConf)) { + addService(service); + } + super.init(hiveConf); // Add a shutdown hook for catching SIGTERM & SIGINT @@ -111,6 +118,15 @@ public void run() { }); } + private List getServices(HiveConf hiveConf) { + try { + return Utilities.getInstances(hiveConf, + ConfVars.HIVE_SERVER2_SERVICE_CLASSES, Service.class); + } catch (Exception e) { + throw new RuntimeException("Failed to instanciate services for hiveserver2", e); + } + } + public static boolean isHTTPTransportMode(HiveConf hiveConf) { String transportMode = System.getenv("HIVE_SERVER2_TRANSPORT_MODE"); if (transportMode == null) {