commit 191bae39a1271c43d10763e950b2015340a7ff67 Author: Bharath Krishna Date: Sun Apr 15 20:15:45 2018 -0700 HIVE-18469 : Introduce separate option to show query on web ui. - Adding option HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e540d023bdda5fbb6152d7ba93c134c0542bf9dd..eedb7301ed0d211cba1168e1bc4ecf0946d8e141 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2821,9 +2821,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal " drawbacks."), HIVE_LOG_EXPLAIN_OUTPUT("hive.log.explain.output", false, - "Whether to log explain output for every query.\n" + - "When enabled, will log EXPLAIN EXTENDED output for the query at INFO log4j log level\n" + - "and in WebUI / Drilldown / Show Query."), + "Whether to log explain output for every query.\n" + + "When enabled, will log EXPLAIN EXTENDED output for the query at INFO log4j log level."), HIVE_EXPLAIN_USER("hive.explain.user", true, "Whether to show explain result at user level.\n" + "When enabled, will log EXPLAIN output for the query at user level. Tez only."), @@ -2983,6 +2982,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "The maximum number of past queries to show in HiverSever2 WebUI."), HIVE_SERVER2_WEBUI_USE_PAM("hive.server2.webui.use.pam", false, "If true, the HiveServer2 WebUI will be secured with PAM."), + HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT("hive.server2.webui.explain.output", false, + "When set to true, the EXPLAIN output for every query is displayed" + + " in the HS2 WebUI / Drilldown / Query Plan tab.\n"), // Tez session settings HIVE_SERVER2_ACTIVE_PASSIVE_HA_ENABLE("hive.server2.active.passive.ha.enable", false, diff --git itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java index b51523fd2574c5b4b8c1dfe114b4e23bd93f2f1a..22f953f4e0ba2c6ccff063085e028987d733807e 100644 --- itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java +++ itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java @@ -25,7 +25,6 @@ import org.apache.hive.service.cli.OperationHandle; import org.apache.hive.service.cli.SessionHandle; import org.apache.hive.service.rpc.thrift.TProtocolVersion; -import org.apache.hive.service.server.HiveServer2; import org.apache.hive.tmpl.QueryProfileTmpl; import org.junit.Assert; import org.junit.Before; @@ -57,10 +56,10 @@ public void setup() { */ @Test public void testQueryDisplay() throws Exception { - HiveSession session = sessionManager.createSession( - new SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8), - TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8, - "testuser", "", "", new HashMap(), false, ""); + HiveSession session = sessionManager + .createSession(new SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8), + TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8, "testuser", "", "", + new HashMap(), false, ""); SessionState.start(conf); OperationHandle opHandle1 = session.executeStatement("show databases", null); @@ -74,23 +73,23 @@ public void testQueryDisplay() throws Exception { Assert.assertEquals(liveSqlOperations.size(), 2); Assert.assertEquals(historicSqlOperations.size(), 0); verifyDDL(liveSqlOperations.get(0), "show databases", opHandle1.getHandleIdentifier().toString(), false); - verifyDDL(liveSqlOperations.get(1),"show tables", opHandle2.getHandleIdentifier().toString(), false); + verifyDDL(liveSqlOperations.get(1), "show tables", opHandle2.getHandleIdentifier().toString(), false); session.closeOperation(opHandle1); liveSqlOperations = sessionManager.getOperationManager().getLiveQueryInfos(); historicSqlOperations = sessionManager.getOperationManager().getHistoricalQueryInfos(); Assert.assertEquals(liveSqlOperations.size(), 1); Assert.assertEquals(historicSqlOperations.size(), 1); - verifyDDL(historicSqlOperations.get(0),"show databases", opHandle1.getHandleIdentifier().toString(), true); - verifyDDL(liveSqlOperations.get(0),"show tables", opHandle2.getHandleIdentifier().toString(), false); + verifyDDL(historicSqlOperations.get(0), "show databases", opHandle1.getHandleIdentifier().toString(), true); + verifyDDL(liveSqlOperations.get(0), "show tables", opHandle2.getHandleIdentifier().toString(), false); session.closeOperation(opHandle2); liveSqlOperations = sessionManager.getOperationManager().getLiveQueryInfos(); historicSqlOperations = sessionManager.getOperationManager().getHistoricalQueryInfos(); Assert.assertEquals(liveSqlOperations.size(), 0); Assert.assertEquals(historicSqlOperations.size(), 2); - verifyDDL(historicSqlOperations.get(1),"show databases", opHandle1.getHandleIdentifier().toString(), true); - verifyDDL(historicSqlOperations.get(0),"show tables", opHandle2.getHandleIdentifier().toString(), true); + verifyDDL(historicSqlOperations.get(1), "show databases", opHandle1.getHandleIdentifier().toString(), true); + verifyDDL(historicSqlOperations.get(0), "show tables", opHandle2.getHandleIdentifier().toString(), true); session.close(); } @@ -100,10 +99,10 @@ public void testQueryDisplay() throws Exception { */ @Test public void testWebUI() throws Exception { - HiveSession session = sessionManager.createSession( - new SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8), - TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8, - "testuser", "", "", new HashMap(), false, ""); + HiveSession session = sessionManager + .createSession(new SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8), + TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8, "testuser", "", "", + new HashMap(), false, ""); SessionState.start(conf); OperationHandle opHandle1 = session.executeStatement("show databases", null); @@ -122,6 +121,35 @@ public void testWebUI() throws Exception { session.close(); } + /** + * Test for the HiveConf option HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT. + */ + @Test + public void checkWebuiExplainOutput() throws Exception { + + //check cases when HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT is set and not set + boolean[] webuiExplainConfValues = new boolean[]{true, false}; + + for (boolean confValue : webuiExplainConfValues) { + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT, confValue); + HiveSession session = sessionManager + .createSession(new SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8), + TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8, "testuser", "", "", + new HashMap(), false, ""); + SessionState.start(conf); + + OperationHandle opHandle = session.executeStatement("show tables", null); + session.closeOperation(opHandle); + //STAGE PLANS is something which will be shown as part of EXPLAIN query + verifyDDLHtml("STAGE PLANS", opHandle.getHandleIdentifier().toString(), confValue); + //Check that the following message is not shown when this option is set + verifyDDLHtml( + "Set configuration hive.server2.webui.explain.output to true to view future query plans", + opHandle.getHandleIdentifier().toString(), !confValue); + session.close(); + } + } + private void verifyDDL(QueryInfo queryInfo, String stmt, String handle, boolean finished) { Assert.assertEquals(queryInfo.getUserName(), "testuser"); @@ -130,8 +158,9 @@ private void verifyDDL(QueryInfo queryInfo, String stmt, String handle, boolean Assert.assertTrue(queryInfo.getBeginTime() > 0 && queryInfo.getBeginTime() <= System.currentTimeMillis()); if (finished) { - Assert.assertTrue(queryInfo.getEndTime() > 0 && queryInfo.getEndTime() >= queryInfo.getBeginTime() - && queryInfo.getEndTime() <= System.currentTimeMillis()); + Assert.assertTrue( + queryInfo.getEndTime() > 0 && queryInfo.getEndTime() >= queryInfo.getBeginTime() + && queryInfo.getEndTime() <= System.currentTimeMillis()); Assert.assertTrue(queryInfo.getRuntime() > 0); } else { Assert.assertNull(queryInfo.getEndTime()); @@ -158,8 +187,8 @@ private void verifyDDL(QueryInfo queryInfo, String stmt, String handle, boolean Assert.assertEquals(tInfo1.getTaskId(), "Stage-0"); Assert.assertEquals(tInfo1.getTaskType(), StageType.DDL); Assert.assertTrue(tInfo1.getBeginTime() > 0 && tInfo1.getBeginTime() <= System.currentTimeMillis()); - Assert.assertTrue(tInfo1.getEndTime() > 0 && tInfo1.getEndTime() >= tInfo1.getBeginTime() && - tInfo1.getEndTime() <= System.currentTimeMillis()); + Assert.assertTrue(tInfo1.getEndTime() > 0 && tInfo1.getEndTime() >= tInfo1.getBeginTime() + && tInfo1.getEndTime() <= System.currentTimeMillis()); Assert.assertEquals(tInfo1.getStatus(), "Success, ReturnVal 0"); } @@ -168,14 +197,17 @@ private void verifyDDL(QueryInfo queryInfo, String stmt, String handle, boolean * assert each element, to make it easier to add UI improvements. */ private void verifyDDLHtml(String stmt, String opHandle) throws Exception { + verifyDDLHtml(stmt, opHandle, true); + } + + private void verifyDDLHtml(String stmt, String opHandle, boolean assertCondition) throws Exception { StringWriter sw = new StringWriter(); - QueryInfo queryInfo = sessionManager.getOperationManager().getQueryInfo( - opHandle); + QueryInfo queryInfo = sessionManager.getOperationManager().getQueryInfo(opHandle); HiveConf hiveConf = sessionManager.getOperationManager().getHiveConf(); new QueryProfileTmpl().render(sw, queryInfo, hiveConf); String html = sw.toString(); + Assert.assertEquals(assertCondition, html.contains(stmt)); - Assert.assertTrue(html.contains(stmt)); Assert.assertTrue(html.contains("testuser")); } diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index a88453c97835db847d74b4b4c3ef318d4d6c0ce5..0f27f15a75249e448ba6c0dbf504dd13afe5b2b5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -689,12 +689,15 @@ public void run() { } } - if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) { + if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT) + || conf.getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT)) { String explainOutput = getExplainOutput(sem, plan, tree); if (explainOutput != null) { - LOG.info("EXPLAIN output for queryid " + queryId + " : " - + explainOutput); - if (conf.isWebUiQueryInfoCacheEnabled()) { + if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) { + LOG.info("EXPLAIN output for queryid " + queryId + " : " + explainOutput); + } + if (conf.isWebUiQueryInfoCacheEnabled() + && conf.getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT)) { queryDisplay.setExplainPlan(explainOutput); } } diff --git service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon index 366198e1470df0aef240f523bc2fa97b6dfadb71..f04d65544098f1919dba231df1e9cff7190e266a 100644 --- service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon +++ service/src/jamon/org/apache/hive/tmpl/QueryProfileTmpl.jamon @@ -204,13 +204,13 @@ org.apache.hadoop.hive.conf.HiveConf;
Explain plan
- <%if hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_LOG_EXPLAIN_OUTPUT) %> + <%if hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_EXPLAIN_OUTPUT) %>
           <% queryInfo.getQueryDisplay() == null ? "Unknown" : queryInfo.getQueryDisplay().getExplainPlan() %>
           
<%else>
-          Set configuration hive.log.explain.output to true to view future query plans
+          Set configuration hive.server2.webui.explain.output to true to view future query plans