diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index a93cc1b7e1..1ac274641d 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -306,6 +306,7 @@ public void run() { } try { + logCompactionParameters(hiveConf); maybeStartCompactorThreads(hiveConf); } catch (Exception e) { throw new RuntimeException(e); @@ -427,6 +428,17 @@ public void run() { ShutdownHookManager.addShutdownHook(() -> hiveServer2.stop()); } + private void logCompactionParameters(HiveConf hiveConf) { + LOG.info("Compaction HS2 parameters:"); + String runWorkerIn = MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.HIVE_METASTORE_RUNWORKER_IN); + LOG.info("hive.metastore.runworker.in = {}", runWorkerIn); + int numWorkers = MetastoreConf.getIntVar(hiveConf, MetastoreConf.ConfVars.COMPACTOR_WORKER_THREADS); + LOG.info("metastore.compactor.worker.threads = {}", numWorkers); + if (runWorkerIn.equals("hs2") && numWorkers < 1) { + LOG.warn("Invalid number of Compactor Worker threads({}) on HS2", numWorkers); + } + } + private WMFullResourcePlan createTestResourcePlan() { WMFullResourcePlan resourcePlan; WMPool pool = new WMPool("testDefault", "llap"); @@ -1056,6 +1068,7 @@ private void maybeStartCompactorThreads(HiveConf hiveConf) throws Exception { Worker w = new Worker(); CompactorThread.initializeAndStartThread(w, hiveConf); } + LOG.info("This HS2 instance will act as Compactor Worker with {} threads", numWorkers); } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 94698e6771..56f838f67d 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -10101,6 +10101,7 @@ public void processContext(ServerContext serverContext, TTransport tTransport, T + maxWorkerThreads); HMSHandler.LOG.info("TCP keepalive = " + tcpKeepAlive); HMSHandler.LOG.info("Enable SSL = " + useSSL); + logCompactionParameters(conf); boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL); HMSHandler.LOG.info("Direct SQL optimization = {}", directSqlEnabled); @@ -10130,6 +10131,39 @@ public void processContext(ServerContext serverContext, TTransport tTransport, T tServer.serve(); } + private static void logCompactionParameters(Configuration conf) { + HMSHandler.LOG.info("Compaction HMS parameters:"); + HMSHandler.LOG + .info("metastore.compactor.initiator.on = {}", MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_INITIATOR_ON)); + HMSHandler.LOG.info("metastore.compactor.worker.threads = {}", + MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_WORKER_THREADS)); + HMSHandler.LOG + .info("hive.metastore.runworker.in = {}", MetastoreConf.getVar(conf, ConfVars.HIVE_METASTORE_RUNWORKER_IN)); + HMSHandler.LOG.info("metastore.compactor.history.reaper.interval = {}", + MetastoreConf.getTimeVar(conf, ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, TimeUnit.MINUTES)); + HMSHandler.LOG.info("metastore.compactor.history.retention.attempted = {}", + MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED)); + HMSHandler.LOG.info("metastore.compactor.history.retention.failed = {}", + MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED)); + HMSHandler.LOG.info("metastore.compactor.history.retention.succeeded = {}", + MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_SUCCEEDED)); + HMSHandler.LOG.info("metastore.compactor.initiator.failed.compacts.threshold = {}", + MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD)); + HMSHandler.LOG.info("metastore.compactor.enable.stats.compression", + MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_MINOR_STATS_COMPRESSION)); + + if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_INITIATOR_ON)) { + LOG.warn("Compactor Initiator is turned Off. Automatic compaction will not be triggered."); + } + + if (MetastoreConf.getVar(conf, MetastoreConf.ConfVars.HIVE_METASTORE_RUNWORKER_IN).equals("metastore")) { + int numThreads = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_WORKER_THREADS); + if (numThreads < 1) { + LOG.warn("Invalid number of Compactor Worker threads({}) on HMS", numThreads); + } + } + } + private static boolean isMetastoreHousekeepingLeader(Configuration conf, String serverHost) { String leaderHost = MetastoreConf.getVar(conf, @@ -10295,6 +10329,7 @@ private static void startCompactorInitiator(Configuration conf) throws Exception MetaStoreThread initiator = instantiateThread("org.apache.hadoop.hive.ql.txn.compactor.Initiator"); initializeAndStartThread(initiator, conf); + LOG.info("This HMS instance will act as a Compactor Initiator."); } } @@ -10305,6 +10340,7 @@ private static void startCompactorWorkers(Configuration conf) throws Exception { instantiateThread("org.apache.hadoop.hive.ql.txn.compactor.Worker"); initializeAndStartThread(worker, conf); } + LOG.info("This HMS instance will act as a Compactor Worker with {} threads", numWorkers); } private static void startCompactorCleaner(Configuration conf) throws Exception { @@ -10312,6 +10348,7 @@ private static void startCompactorCleaner(Configuration conf) throws Exception { MetaStoreThread cleaner = instantiateThread("org.apache.hadoop.hive.ql.txn.compactor.Cleaner"); initializeAndStartThread(cleaner, conf); + LOG.info("This HMS instance will act as a Compactor Cleaner."); } } @@ -10412,4 +10449,5 @@ private static String getHostname() { try {return "" + InetAddress.getLocalHost();} catch(UnknownHostException uhe) {return "" + uhe;} } + }