diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 97fe7bc..35d06d6 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1902,6 +1902,8 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "HTTP/_HOST@EXAMPLE.COM", "The HiveServer2 WebUI SPNEGO service principal.\n" + "The special string _HOST will be replaced automatically with \n" + "the value of hive.server2.webui.host or the correct host name."), + HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES("hive.server2.webui.max.historic.queries", 25, + "The max old queries HiveServer2 WebUI will show"), // Tez session settings HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "", diff --git a/pom.xml b/pom.xml index 2066518..9302195 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ 1.4 10.10.2.0 3.1.0 - 14.0.1 + 15.0 2.4.4 2.6.0 ${basedir}/${hive.path.to.root}/testutils/hadoop diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 75187cf..de55c35 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -158,6 +158,9 @@ // HS2 operation handle guid string private String operationId; + //For WebUI + private String savedQueryString; + private boolean checkConcurrency() { boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY); if (!supportConcurrency) { @@ -384,6 +387,7 @@ public int compile(String command, boolean resetTaskIds) { } catch (Exception e) { LOG.warn("WARNING! Query command could not be redacted." + e); } + this.savedQueryString = queryStr; //holder for parent command type/string when executing reentrant queries QueryState queryState = new QueryState(); @@ -1943,6 +1947,11 @@ public String getErrorMsg() { return errorMessage; } + + public String getQueryString() { + return savedQueryString == null ? "Unknown" : savedQueryString; + } + /** * Set the HS2 operation handle's guid string * @param opId base64 encoded guid string diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java index 113eddf..ee3c8ea 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java @@ -74,6 +74,7 @@ private long operationTimeout; private volatile long lastAccessTime; + private long beginTime; protected static final EnumSet DEFAULT_FETCH_ORIENTATION_SET = EnumSet.of(FetchOrientation.FETCH_NEXT,FetchOrientation.FETCH_FIRST); @@ -89,6 +90,7 @@ protected Operation(HiveSession parentSession, Map confOverlay, } this.runAsync = runInBackground; this.opHandle = new OperationHandle(opType, parentSession.getProtocolVersion()); + beginTime = System.currentTimeMillis(); lastAccessTime = System.currentTimeMillis(); operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS); @@ -407,4 +409,12 @@ protected void setMetrics(OperationState state) { } } } + + public long getBeginTime() { + return beginTime; + } + + protected OperationState getState() { + return state; + } } diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java index 92135cd..d2a3877 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java +++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java @@ -22,10 +22,14 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; +import com.google.common.collect.EvictingQueue; import org.apache.hadoop.hive.common.metrics.common.Metrics; import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; @@ -61,6 +65,10 @@ private final ConcurrentHashMap handleToOperation = new ConcurrentHashMap(); + //for displaying historical queries on WebUI + private Queue historicSqlOperations; + + public OperationManager() { super(OperationManager.class.getSimpleName()); } @@ -73,6 +81,11 @@ public synchronized void init(HiveConf hiveConf) { } else { LOG.debug("Operation level logging is turned off"); } + if ((hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) && + hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) { + historicSqlOperations = EvictingQueue.create( + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES)); + } super.init(hiveConf); } @@ -174,6 +187,7 @@ private Operation removeTimedOutOperation(OperationHandle operationHandle) { Operation operation = handleToOperation.get(operationHandle); if (operation != null && operation.isTimedOut(System.currentTimeMillis())) { handleToOperation.remove(operationHandle, operation); + cacheOldOperationInfo(operation); return operation; } return null; @@ -184,7 +198,9 @@ private void addOperation(Operation operation) { } private Operation removeOperation(OperationHandle opHandle) { - return handleToOperation.remove(opHandle); + Operation result = handleToOperation.remove(opHandle); + cacheOldOperationInfo(result); + return result; } public OperationStatus getOperationStatus(OperationHandle opHandle) @@ -313,4 +329,33 @@ public OperationLog getOperationLogByThread() { } return removed; } + + //Cache a number of historical operation info, at max number of + //hive.server2.webui.max.historic.queries. + private void cacheOldOperationInfo(Operation oldOperation) { + if ((getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) && + getHiveConf().getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) { + if (oldOperation instanceof SQLOperation) { + SQLOperation query = (SQLOperation) oldOperation; + synchronized (historicSqlOperations) { + historicSqlOperations.add(query.getSQLOperationInfo()); + } + } + } + } + + /** + * @return a number of historical SQLOperation info, at max number of + * hive.server2.webui.max.historic.queries + */ + public List getHistoricalSQLOpInfo() { + List result = new LinkedList<>(); + synchronized (historicSqlOperations) { + Iterator opIterator = historicSqlOperations.iterator(); + while (opIterator.hasNext()) { + result.add(opIterator.next()); + } + } + return result; + } } diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index c8a69b9..bb650b1 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -163,7 +163,7 @@ public void prepare(HiveConf sqlOperationConf) throws HiveSQLException { } public String getQueryStr() { - return driver == null || driver.getPlan() == null ? "Unknown" : driver.getPlan().getQueryStr(); + return driver == null ? "Unknown" : driver.getQueryString(); } private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException { @@ -199,6 +199,7 @@ private void runQuery(HiveConf sqlOperationConf) throws HiveSQLException { public void runInternal() throws HiveSQLException { setState(OperationState.PENDING); final HiveConf opConfig = getConfigForOperation(); + prepare(opConfig); if (!shouldRunAsync()) { runQuery(opConfig); @@ -304,6 +305,7 @@ private void cleanup(OperationState state) throws HiveSQLException { backgroundHandle.cancel(true); } } + if (driver != null) { driver.close(); driver.destroy(); @@ -479,4 +481,27 @@ public HiveConf getConfigForOperation() throws HiveSQLException { } return sqlOperationConf; } + + /** + * Get summary information of this SQLOperation for display in WebUI, + * if WebUI is enabled. + */ + public SQLOperationInfo getSQLOperationInfo() { + try { + HiveConf conf = getParentSession().getHiveConf(); + if ((conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT) != 0) && + conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0) { + return new SQLOperationInfo( + getParentSession().getUserName(), + driver.getQueryString(), + getConfigForOperation().getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE), + getState(), + (int) (System.currentTimeMillis() - getBeginTime()) / 1000, + System.currentTimeMillis()); + } + } catch (HiveSQLException e) { + LOG.warn("Error calcluating SQL Operation Info for webui", e); + } + return null; + } } diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java new file mode 100644 index 0000000..179f6dd --- /dev/null +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperationInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.service.cli.operation; + +import org.apache.hive.service.cli.OperationState; + +/** + * Used to display some info in the HS2 WebUI. + */ +public class SQLOperationInfo { + public String userName; + public String queryStr; + public String executionEngine; + public OperationState endState; //state before CLOSED (one of CANCELLED, FINISHED, ERROR) + public int elapsedTime; + public long endTime; + + public SQLOperationInfo( + String userName, + String queryStr, + String executionEngine, + OperationState endState, + int elapsedTime, + long endTime + ) { + this.userName = userName; + this.queryStr = queryStr; + this.executionEngine = executionEngine; + this.endState = endState; + this.elapsedTime = elapsedTime; + this.endTime = endTime; + } +} diff --git a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp index a91b008..f1ecd72 100644 --- a/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp +++ b/service/src/resources/hive-webapps/hiveserver2/hiveserver2.jsp @@ -24,11 +24,14 @@ import="org.apache.hive.common.util.HiveVersionInfo" import="org.apache.hive.service.cli.operation.Operation" import="org.apache.hive.service.cli.operation.SQLOperation" + import="org.apache.hive.service.cli.operation.SQLOperationInfo" import="org.apache.hive.service.cli.session.SessionManager" import="org.apache.hive.service.cli.session.HiveSession" import="javax.servlet.ServletContext" import="java.util.Collection" import="java.util.Date" + import="java.util.Set" + import="java.util.List" %> <% @@ -145,7 +148,7 @@ for (Operation operation: operations) { <%= query.getQueryStr() %> <%= query.getConfigForOperation().getVar(ConfVars.HIVE_EXECUTION_ENGINE) %> <%= query.getStatus().getState() %> - <%= (currentTime - query.getLastAccessTime())/1000 %> + <%= (currentTime - query.getBeginTime())/1000 %> <% } @@ -156,6 +159,43 @@ for (Operation operation: operations) { + + +
+

Last Max <%= conf.get(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES.varname) %> Completed Queries

+ + + + + + + + + +<% +queries = 0; +List sqlOperations = sessionManager.getOperationManager().getHistoricalSQLOpInfo(); +for (SQLOperationInfo sqlOperation: sqlOperations) { + queries++; +%> + + + + + + + + +<% +} + +%> + + + +
User NameQueryExecution EngineStateElapsed Time (s)End Time
<%= sqlOperation.userName %><%= sqlOperation.queryStr %><%= sqlOperation.executionEngine %><%= sqlOperation.endState %><%= sqlOperation.elapsedTime %><%= new Date(sqlOperation.endTime) %>
Total number of queries: <%= queries %>
+
+ <% } %>