diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2a52fd1..f25c2a5 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -92,6 +92,7 @@ public class HiveConf extends Configuration { HiveConf.ConfVars.METASTORE_EVENT_CLEAN_FREQ, HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION, HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, + HiveConf.ConfVars.METASTORE_LISTEN_PORT, }; /** @@ -245,6 +246,7 @@ public class HiveConf extends Configuration { METASTORE_DETACH_ALL_ON_COMMIT("javax.jdo.option.DetachAllOnCommit", true), METASTORE_NON_TRANSACTIONAL_READ("javax.jdo.option.NonTransactionalRead", true), METASTORE_CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", "APP"), + METASTORE_LISTEN_PORT("hive.metastore.listen.port",9803), // CLI CLIIGNOREERRORS("hive.cli.errors.ignore", false), @@ -507,6 +509,8 @@ public class HiveConf extends Configuration { // Whether to delete the scratchdir while startup HIVE_START_CLEANUP_SCRATCHDIR("hive.start.cleanup.scratchdir", false), HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false), + // hive server + HIVE_SERVER_LISTEN_PORT("hive.server.listen.port", 10000), ; public final String varname; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index ffce4fe..e790245 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -128,7 +128,8 @@ public class HiveMetaStore extends ThriftHiveMetastore { /** * default port on which to start the Hive server */ - private static final int DEFAULT_HIVE_METASTORE_PORT = 9083; + private static final int DEFAULT_HIVE_METASTORE_PORT = + HiveConf.ConfVars.METASTORE_LISTEN_PORT.defaultIntVal; private static HadoopThriftAuthBridge.Server saslServer; private static boolean useSasl; @@ -3591,12 +3592,6 @@ public class HiveMetaStore extends ThriftHiveMetastore { // deprecated (old style) naked args... if (commandLine.hasOption('p')) { port = Integer.parseInt(commandLine.getOptionValue('p')); - } else { - // legacy handling - String metastorePort = System.getenv("METASTORE_PORT"); - if (metastorePort != null) { - port = Integer.parseInt(metastorePort); - } } } } @@ -3625,12 +3620,6 @@ public class HiveMetaStore extends ThriftHiveMetastore { } try { - String msg = "Starting hive metastore on port " + cli.port; - HMSHandler.LOG.info(msg); - if (cli.isVerbose()) { - System.err.println(msg); - } - HiveConf conf = new HiveConf(HMSHandler.class); // set all properties specified on the command line @@ -3638,7 +3627,16 @@ public class HiveMetaStore extends ThriftHiveMetastore { conf.set((String) item.getKey(), (String) item.getValue()); } - startMetaStore(cli.port, ShimLoader.getHadoopThriftAuthBridge(), conf); + applyCliOptionsToHiveConf(cli, conf); + + String msg = "Starting hive metastore on port " + + conf.getIntVar(HiveConf.ConfVars.METASTORE_LISTEN_PORT); + HMSHandler.LOG.info(msg); + if (cli.isVerbose()) { + System.err.println(msg); + } + + startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), conf); } catch (Throwable t) { // Catch the exception, log it and rethrow it. HMSHandler.LOG @@ -3647,15 +3645,30 @@ public class HiveMetaStore extends ThriftHiveMetastore { } } + private static void applyCliOptionsToHiveConf(HiveMetastoreCli cli, HiveConf conf) { + if(cli.port != DEFAULT_HIVE_METASTORE_PORT) { + // -p option is set + conf.setIntVar(HiveConf.ConfVars.METASTORE_LISTEN_PORT, cli.port); + } + if (conf.getIntVar(HiveConf.ConfVars.METASTORE_LISTEN_PORT) != DEFAULT_HIVE_METASTORE_PORT) { + // neither -p cli option nor HiveConf is set + // legacy handling + String metastorePort = System.getenv("METASTORE_PORT"); + if (metastorePort != null) { + conf.setIntVar(HiveConf.ConfVars.METASTORE_LISTEN_PORT, Integer.parseInt(metastorePort)); + } + } + } + /** * Start Metastore based on a passed {@link HadoopThriftAuthBridge} * @param port * @param bridge * @throws Throwable */ - public static void startMetaStore(int port, HadoopThriftAuthBridge bridge) + public static void startMetaStore(HadoopThriftAuthBridge bridge) throws Throwable { - startMetaStore(port, bridge, new HiveConf(HMSHandler.class)); + startMetaStore(bridge, new HiveConf(HMSHandler.class)); } /** @@ -3665,9 +3678,10 @@ public class HiveMetaStore extends ThriftHiveMetastore { * @param hiveconf configuration overrides * @throws Throwable */ - public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, - HiveConf conf) throws Throwable { + public static void startMetaStore(HadoopThriftAuthBridge bridge, + HiveConf oldConf) throws Throwable { try { + HiveConf conf = new HiveConf(oldConf, HMSHandler.class); HMSHandler handler = new HMSHandler("new db based metaserver", conf); // Server will create new threads up to max as necessary. After an idle @@ -3677,6 +3691,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { int maxWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMAXTHREADS); boolean tcpKeepAlive = conf.getBoolVar(HiveConf.ConfVars.METASTORE_TCP_KEEP_ALIVE); useSasl = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL); + int port = conf.getIntVar(HiveConf.ConfVars.METASTORE_LISTEN_PORT); TServerTransport serverTransport = tcpKeepAlive ? new TServerSocketKeepAlive(port) : new TServerSocket(port); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index cd533b6..ddbadab 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -730,22 +730,23 @@ public class MetaStoreUtils { } - public static void startMetaStore(final int port, - final HadoopThriftAuthBridge bridge) throws Exception { + public static void startMetaStore(final HadoopThriftAuthBridge bridge, final HiveConf conf) + throws Exception { Thread thread = new Thread(new Runnable() { @Override public void run() { try { - HiveMetaStore.startMetaStore(port, bridge); + HiveMetaStore.startMetaStore(bridge, conf); } catch (Throwable e) { - LOG.error("Metastore Thrift Server threw an exception...",e); + LOG.error("Metastore Thrift Server threw an exception...", e); } } }); thread.setDaemon(true); thread.start(); - loopUntilHMSReady(port); + loopUntilHMSReady(conf.getIntVar(HiveConf.ConfVars.METASTORE_LISTEN_PORT)); } + /** * A simple connect test to make sure that the metastore is up * @throws Exception diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java index a089269..e591191 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -32,15 +33,14 @@ import org.apache.hadoop.hive.shims.ShimLoader; public class TestMetaStoreAuthorization extends TestCase { - protected HiveConf conf = new HiveConf(); - - private final int port = 10000; + protected HiveConf conf = new HiveConf(HMSHandler.class); public void setup() throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS.varname, "true"); conf.setBoolVar(ConfVars.METASTORE_MODE, false); - conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); + conf.setVar(HiveConf.ConfVars.METASTOREURIS, + "thrift://localhost:" + conf.getIntVar(HiveConf.ConfVars.METASTORE_LISTEN_PORT)); conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3); conf.setIntVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, 60); } @@ -71,7 +71,7 @@ public class TestMetaStoreAuthorization extends TestCase { public void testMetaStoreAuthorization() throws Exception { setup(); - MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); + MetaStoreUtils.startMetaStore(ShimLoader.getHadoopThriftAuthBridge(),conf); HiveMetaStoreClient client = new HiveMetaStoreClient(conf); FileSystem fs = null; diff --git a/service/src/java/org/apache/hadoop/hive/service/HiveServer.java b/service/src/java/org/apache/hadoop/hive/service/HiveServer.java index a2d599f..cd977d5 100644 --- a/service/src/java/org/apache/hadoop/hive/service/HiveServer.java +++ b/service/src/java/org/apache/hadoop/hive/service/HiveServer.java @@ -34,9 +34,9 @@ import java.util.Properties; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.common.ServerUtils; import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.common.LogUtils.LogInitializationException; +import org.apache.hadoop.hive.common.ServerUtils; import org.apache.hadoop.hive.common.cli.CommonCliOptions; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore; @@ -61,8 +61,7 @@ import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TServerTransport; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; + import com.facebook.fb303.fb_status; /** @@ -74,7 +73,8 @@ public class HiveServer extends ThriftHive { /** * default port on which to start the Hive server */ - private static final int DEFAULT_HIVE_SERVER_PORT = 10000; + private static final int DEFAULT_HIVE_SERVER_PORT = + HiveConf.ConfVars.HIVE_SERVER_LISTEN_PORT.defaultIntVal; /** * default minimum number of threads serving the Hive server @@ -632,12 +632,6 @@ public class HiveServer extends ThriftHive { // deprecated (old style) naked args... if (commandLine.hasOption('p')) { port = Integer.parseInt(commandLine.getOptionValue('p')); - } else { - // legacy handling - String hivePort = System.getenv("HIVE_PORT"); - if (hivePort != null) { - port = Integer.parseInt(hivePort); - } } if (commandLine.hasOption(OPTION_MIN_WORKER_THREADS)) { minWorkerThreads = Integer.parseInt( @@ -648,8 +642,23 @@ public class HiveServer extends ThriftHive { commandLine.getOptionValue(OPTION_MAX_WORKER_THREADS)); } } + + public void applyHiveConf(HiveConf conf) { + if (port == DEFAULT_HIVE_SERVER_PORT) { + // applying configuration only when "-p" is not. + port = conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER_LISTEN_PORT); + } + + if (port == DEFAULT_HIVE_SERVER_PORT) { + // legacy handling + String hivePort = System.getenv("HIVE_PORT"); + if (hivePort != null) { + port = Integer.parseInt(hivePort); + } + } + } } - + public static void main(String[] args) { try { HiveServerCli cli = new HiveServerCli(); @@ -670,13 +679,15 @@ public class HiveServer extends ThriftHive { HiveConf conf = new HiveConf(HiveServerHandler.class); ServerUtils.cleanUpScratchDir(conf); - TServerTransport serverTransport = new TServerSocket(cli.port); // set all properties specified on the command line for (Map.Entry item : hiveconf.entrySet()) { conf.set((String) item.getKey(), (String) item.getValue()); } + cli.applyHiveConf(conf); + + TServerTransport serverTransport = new TServerSocket(cli.port); ThriftHiveProcessorFactory hfactory = new ThriftHiveProcessorFactory(null, conf); @@ -686,7 +697,7 @@ public class HiveServer extends ThriftHive { .protocolFactory(new TBinaryProtocol.Factory()) .minWorkerThreads(cli.minWorkerThreads) .maxWorkerThreads(cli.maxWorkerThreads); - + TServer server = new TThreadPoolServer(sargs); String msg = "Starting hive server on port " + cli.port diff --git a/shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java b/shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java index aadb34d..52bdfbd 100644 --- a/shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java +++ b/shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java @@ -117,7 +117,9 @@ public class TestHadoop20SAuthBridge extends TestCase { System.getProperty("test.build.data", "/tmp")).toString()); conf = new HiveConf(TestHadoop20SAuthBridge.class); conf.setBoolVar(ConfVars.METASTORE_MODE, false); - MetaStoreUtils.startMetaStore(port, new MyHadoopThriftAuthBridge20S()); + conf.setIntVar(ConfVars.METASTORE_LISTEN_PORT, port); + + MetaStoreUtils.startMetaStore(new MyHadoopThriftAuthBridge20S(),conf); } public void testSaslWithHiveMetaStore() throws Exception {