diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 1631e2d..e59f63b 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1596,8 +1596,13 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "${system:java.io.tmpdir}" + File.separator + "${system:user.name}" + File.separator + "operation_logs", "Top level directory where operation logs are stored if logging functionality is enabled"), - HIVE_SERVER2_LOGGING_OPERATION_VERBOSE("hive.server2.logging.operation.verbose", false, - "When true, HS2 operation logs available for clients will be verbose"), + HIVE_SERVER2_LOGGING_OPERATION_LEVEL("hive.server2.logging.operation.level", "EXECUTION", + new StringSet("NONE", "EXECUTION", "PERFORMANCE", "VERBOSE"), + "HS2 operation logging mode available to clients to be set at session level.\n" + + " NONE: Ignore any logging\n" + + " EXECUTION: Log completion of map-reduce tasks\n" + + " PERFORMANCE: Execution + Performance logs \n" + + " VERBOSE: All logs" ), // logging configuration HIVE_LOG4J_FILE("hive.log4j.file", "", "Hive log4j configuration file.\n" + diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index 2c85877..57dcc2a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -106,7 +106,7 @@ public TestJdbcDriver2() { public static void setUpBeforeClass() throws SQLException, ClassNotFoundException{ Class.forName(driverName); Connection con1 = getConnection("default"); - System.setProperty(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_VERBOSE.varname, "" + true); + System.setProperty(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose"); Statement stmt1 = con1.createStatement(); assertNotNull("Statement is null", stmt1); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java index 571bbbc..f05ab77 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java @@ -20,6 +20,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.io.IOUtils; import java.io.*; @@ -36,10 +37,16 @@ private final String operationName; private final LogFile logFile; + private HiveConf hiveConf; - public OperationLog(String name, File file) throws FileNotFoundException{ + public OperationLog(String name, File file, HiveConf hiveConf) throws FileNotFoundException{ operationName = name; logFile = new LogFile(file); + this.hiveConf = hiveConf; + } + + public HiveConf getHiveConf() { + return hiveConf; } /** diff --git a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java index 876ade8..2deb3f1 100644 --- a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java +++ b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java @@ -18,6 +18,9 @@ package org.apache.hive.service.cli; +import org.apache.log4j.Layout; +import org.apache.log4j.PatternLayout; + /** * CLIServiceUtils. * @@ -26,6 +29,10 @@ private static final char SEARCH_STRING_ESCAPE = '\\'; + public static final Layout verboseLayout = new PatternLayout( + "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"); + public static final Layout nonVerboseLayout = new PatternLayout( + "%-5p : %m%n"); /** * Convert a SQL search pattern into an equivalent Java Regex. @@ -66,4 +73,19 @@ public static String patternToRegex(String pattern) { } } + public static boolean isVerboseLoggingMode(String mode) { + return mode.equalsIgnoreCase("verbose"); + } + + public static boolean isNoneLoggingMode(String mode) { + return mode.equalsIgnoreCase("none"); + } + + public static boolean isPerformanceLoggingMode(String mode) { + return mode.equalsIgnoreCase("performance"); + } + + public static boolean isExecutionLoggingMode(String mode) { + return mode.equalsIgnoreCase("execution"); + } } diff --git a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java index 4737785..ca56c09 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java +++ b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java @@ -20,8 +20,10 @@ import java.io.CharArrayWriter; import java.util.regex.Pattern; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.session.OperationLog; +import org.apache.hive.service.cli.CLIServiceUtils; import org.apache.log4j.Layout; import org.apache.log4j.Logger; import org.apache.log4j.WriterAppender; @@ -36,6 +38,8 @@ public class LogDivertAppender extends WriterAppender { private static final Logger LOG = Logger.getLogger(LogDivertAppender.class.getName()); private final OperationManager operationManager; + private boolean isVerbose; + private Layout verboseLayout; /** * A log filter that filters messages coming from the logger with the given names. @@ -45,18 +49,67 @@ * White list filter is used for less verbose log collection */ private static class NameFilter extends Filter { - private final Pattern namePattern; - private final boolean excludeMatches; + private Pattern namePattern; + private boolean excludeMatches; + private String loggingMode; + private OperationManager operationManager; - public NameFilter(boolean isExclusionFilter, String [] loggerNames) { + private static final Pattern verboseNamePattern = Pattern.compile(Joiner.on("|"). + join(new String[] {LOG.getName(), OperationLog.class.getName(), + OperationManager.class.getName()})); + private static final Pattern executionNamePattern = Pattern.compile(Joiner.on("|"). + join(new String[] {"org.apache.hadoop.mapreduce.JobSubmitter", + "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(), + "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"})); + private static final Pattern performanceNamePattern = Pattern.compile(Joiner.on("|"). + join(new String[] {"org.apache.hadoop.mapreduce.JobSubmitter", + "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(), + "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor", + new HiveConf().getVar(HiveConf.ConfVars.HIVE_PERF_LOGGER)})); + + private void setCurrentNamePattern(String mode) { + if (CLIServiceUtils.isVerboseLoggingMode(mode)) { + this.namePattern = verboseNamePattern; + } else if (CLIServiceUtils.isExecutionLoggingMode(mode)) { + this.namePattern = executionNamePattern; + } else if (CLIServiceUtils.isPerformanceLoggingMode(mode)) { + this.namePattern = performanceNamePattern; + } + } + + public NameFilter(boolean isExclusionFilter, + String loggingMode, OperationManager op) { this.excludeMatches = isExclusionFilter; - String matchRegex = Joiner.on("|").join(loggerNames); - this.namePattern = Pattern.compile(matchRegex); + this.operationManager = op; + this.loggingMode = loggingMode; + setCurrentNamePattern(loggingMode); } @Override public int decide(LoggingEvent ev) { + OperationLog log = operationManager.getOperationLogByThread(); + + if (log == null) { + return Filter.DENY; + } + + String currentLoggingMode = log.getHiveConf(). + getVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL); + + // If logging is disabled, deny everything. + if (CLIServiceUtils.isNoneLoggingMode(currentLoggingMode)) { + return Filter.DENY; + } + // Look at the current session's setting + // and set the pattern and excludeMatches accordingly. + if (!currentLoggingMode.equalsIgnoreCase(loggingMode)) { + loggingMode = currentLoggingMode; + this.excludeMatches = CLIServiceUtils.isVerboseLoggingMode(loggingMode); + setCurrentNamePattern(loggingMode); + } + boolean isMatch = namePattern.matcher(ev.getLoggerName()).matches(); + if (excludeMatches == isMatch) { // Deny if this is black-list filter (excludeMatches = true) and it // matched @@ -70,25 +123,44 @@ public int decide(LoggingEvent ev) { /** This is where the log message will go to */ private final CharArrayWriter writer = new CharArrayWriter(); - public LogDivertAppender(Layout layout, OperationManager operationManager, boolean isVerbose) { + public LogDivertAppender(Layout layout, OperationManager operationManager, String loggingMode) { + isVerbose = CLIServiceUtils.isVerboseLoggingMode(loggingMode); + setLayout(layout); setWriter(writer); setName("LogDivertAppender"); this.operationManager = operationManager; - + this.verboseLayout = isVerbose ? layout : CLIServiceUtils.verboseLayout; if (isVerbose) { // Filter out messages coming from log processing classes, or we'll run an // infinite loop. - String[] exclLoggerNames = { LOG.getName(), OperationLog.class.getName(), - OperationManager.class.getName() }; - addFilter(new NameFilter(true, exclLoggerNames)); + addFilter(new NameFilter(true, loggingMode, operationManager)); } else { // in non verbose mode, show only select logger messages - String[] inclLoggerNames = { "org.apache.hadoop.mapreduce.JobSubmitter", - "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(), - "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}; - addFilter(new NameFilter(false, inclLoggerNames)); + addFilter(new NameFilter(false, loggingMode, operationManager)); + } + } + + @Override + public void doAppend(LoggingEvent event) { + OperationLog log = operationManager.getOperationLogByThread(); + + // Set current layout depending on the verbose/non-verbose mode. + if (log != null) { + boolean isCurrModeVerbose = CLIServiceUtils.isVerboseLoggingMode(log.getHiveConf(). + getVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL)); + // If there is a logging level change from verbose->non-verbose or vice-versa since + // the last subAppend call, change the layout to preserve consistency. + if (isCurrModeVerbose ^ isVerbose) { + isVerbose = isCurrModeVerbose; + if (isVerbose) { + setLayout(verboseLayout); + } else { + setLayout(CLIServiceUtils.nonVerboseLayout); + } + } } + super.doAppend(event); } /** diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java index d85db8a..19153b6 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java @@ -210,7 +210,7 @@ protected void createOperationLog() { // create OperationLog object with above log file try { - operationLog = new OperationLog(opHandle.toString(), operationLogFile); + operationLog = new OperationLog(opHandle.toString(), operationLogFile, parentSession.getHiveConf()); } catch (FileNotFoundException e) { LOG.warn("Unable to instantiate OperationLog object for operation: " + opHandle, e); diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java index 6cae8a8..675b9f0 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java +++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.ql.session.OperationLog; import org.apache.hive.service.AbstractService; +import org.apache.hive.service.cli.CLIServiceUtils; import org.apache.hive.service.cli.FetchOrientation; import org.apache.hive.service.cli.HiveSQLException; import org.apache.hive.service.cli.OperationHandle; @@ -45,7 +46,6 @@ import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Layout; import org.apache.log4j.Logger; -import org.apache.log4j.PatternLayout; /** * OperationManager. @@ -66,8 +66,8 @@ public OperationManager() { public synchronized void init(HiveConf hiveConf) { this.hiveConf = hiveConf; if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED)) { - boolean isVerbose = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_VERBOSE); - initOperationLogCapture(isVerbose); + initOperationLogCapture(hiveConf.getVar( + HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL)); } else { LOG.debug("Operation level logging is turned off"); } @@ -86,7 +86,7 @@ public synchronized void stop() { super.stop(); } - private void initOperationLogCapture(boolean isVerbose) { + private void initOperationLogCapture(String loggingMode) { // There should be a ConsoleAppender. Copy its Layout. Logger root = Logger.getRootLogger(); Layout layout = null; @@ -100,19 +100,16 @@ private void initOperationLogCapture(boolean isVerbose) { } } - final String VERBOSE_PATTERN = "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"; - final String NONVERBOSE_PATTERN = "%-5p : %m%n"; - - if (isVerbose) { + if (CLIServiceUtils.isVerboseLoggingMode(loggingMode)) { if (layout == null) { - layout = new PatternLayout(VERBOSE_PATTERN); + layout = CLIServiceUtils.verboseLayout; LOG.info("Cannot find a Layout from a ConsoleAppender. Using default Layout pattern."); } } else { - layout = new PatternLayout(NONVERBOSE_PATTERN); + layout = CLIServiceUtils.nonVerboseLayout; } // Register another Appender (with the same layout) that talks to us. - Appender ap = new LogDivertAppender(layout, this, isVerbose); + Appender ap = new LogDivertAppender(layout, this, loggingMode); root.addAppender(ap); } diff --git a/service/src/test/org/apache/hive/service/cli/operation/TestOperationLoggingAPI.java b/service/src/test/org/apache/hive/service/cli/operation/TestOperationLoggingAPI.java index 42bdf21..62f360f 100644 --- a/service/src/test/org/apache/hive/service/cli/operation/TestOperationLoggingAPI.java +++ b/service/src/test/org/apache/hive/service/cli/operation/TestOperationLoggingAPI.java @@ -48,6 +48,7 @@ private ThriftCLIServiceClient client; private SessionHandle sessionHandle; private final String sql = "select * from " + tableName; + private final String sqlCntStar = "select count(*) from " + tableName; private final String[] expectedLogs = { "Parsing command", "Parse Completed", @@ -55,11 +56,17 @@ "Semantic Analysis Completed", "Starting command" }; + private final String[] expectedLogsExecution = { + "Number of reduce tasks determined at compile time", + "number of splits", + "Submitting tokens for job", + "Ended Job" + }; @BeforeClass public static void setUpBeforeClass() { hiveConf = new HiveConf(); - hiveConf.setBoolean(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_VERBOSE.varname, true); + hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose"); } /** @@ -170,6 +177,35 @@ public void testFetchResultsOfLogWithOrientation() throws Exception { } @Test + public void testFetchResultsOfLogWithExecutionMode() throws Exception { + String queryString = "set hive.server2.logging.operation.level=execution"; + client.executeStatement(sessionHandle, queryString, null); + // verify whether the sql operation log is generated and fetch correctly. + OperationHandle operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null); + RowSet rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 1000, + FetchType.LOG); + verifyFetchedLog(rowSetLog, expectedLogsExecution); + // Restore everything to default setup to avoid discrepancy between junit test runs + String queryString2 = "set hive.server2.logging.operation.level=verbose"; + client.executeStatement(sessionHandle, queryString2, null); + } + + @Test + public void testFetchResultsOfLogWithNoneMode() throws Exception { + String queryString = "set hive.server2.logging.operation.level=none"; + client.executeStatement(sessionHandle, queryString, null); + // verify whether the sql operation log is generated and fetch correctly. + OperationHandle operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null); + RowSet rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 1000, + FetchType.LOG); + // We should not get any rows. + assert(rowSetLog == null || rowSetLog.numRows() == 0); + // Restore everything to default setup to avoid discrepancy between junit test runs + String queryString2 = "set hive.server2.logging.operation.level=verbose"; + client.executeStatement(sessionHandle, queryString2, null); + } + + @Test public void testFetchResultsOfLogCleanup() throws Exception { // Verify cleanup functionality. // Open a new session, since this case needs to close the session in the end. @@ -260,6 +296,20 @@ private void verifyFetchedLog(RowSet rowSet) { verifyFetchedLog(logs); } + private void verifyFetchedLog(RowSet rowSet, String[] el) { + StringBuilder stringBuilder = new StringBuilder(); + + for (Object[] row : rowSet) { + stringBuilder.append(row[0]); + } + + String logs = stringBuilder.toString(); + + for (String log : el) { + Assert.assertTrue("Checking for presence of " + log, logs.contains(log)); + } + } + private void verifyFetchedLog(String logs) { for (String log : expectedLogs) { Assert.assertTrue("Checking for presence of " + log, logs.contains(log));