diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index f3e40eb6dc..790b354150 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -574,7 +574,7 @@ SPARK_CREATE_CLIENT_INTERRUPTED(30040, "Interrupted while creating Spark client for session {0}", true), SPARK_CREATE_CLIENT_ERROR(30041, - "Failed to create Spark client for Spark session {0}", true), + "Failed to create Spark client for Spark session {0}: {1}", true), SPARK_CREATE_CLIENT_INVALID_RESOURCE_REQUEST(30042, "Failed to create Spark client due to invalid resource request: {0}", true), SPARK_CREATE_CLIENT_CLOSED_SESSION(30043, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java index 2d5d03ee71..2d16c9a4d0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java @@ -71,7 +71,6 @@ private HiveSparkClient hiveSparkClient; private Path scratchDir; private final Object dirLock = new Object(); - private String matchedString = null; public SparkSessionImpl() { sessionId = makeSessionId(); @@ -195,6 +194,7 @@ private static void initErrorPatterns() { @VisibleForTesting HiveException getHiveException(Throwable e) { Throwable oe = e; + StringBuilder matchedString = new StringBuilder(); while (e != null) { if (e instanceof TimeoutException) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_TIMEOUT); @@ -202,31 +202,26 @@ HiveException getHiveException(Throwable e) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INTERRUPTED, sessionId); } else if (e instanceof RuntimeException) { String sts = Throwables.getStackTraceAsString(e); - if (matches(sts, AM_TIMEOUT_ERR)) { + if (matches(sts, AM_TIMEOUT_ERR, matchedString)) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_TIMEOUT); - } else if (matches(sts, UNKNOWN_QUEUE_ERR) || matches(sts, STOPPED_QUEUE_ERR)) { - return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_QUEUE, matchedString); - } else if (matches(sts, FULL_QUEUE_ERR)) { - return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_QUEUE_FULL, matchedString); - } else if (matches(sts, INVALILD_MEM_ERR) || matches(sts, INVALID_CORE_ERR)) { + } else if (matches(sts, UNKNOWN_QUEUE_ERR, matchedString) || matches(sts, STOPPED_QUEUE_ERR, matchedString)) { + return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_QUEUE, matchedString.toString()); + } else if (matches(sts, FULL_QUEUE_ERR, matchedString)) { + return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_QUEUE_FULL, matchedString.toString()); + } else if (matches(sts, INVALILD_MEM_ERR, matchedString) || matches(sts, INVALID_CORE_ERR, matchedString)) { return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_INVALID_RESOURCE_REQUEST, - matchedString); + matchedString.toString()); } else { - return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId); + return new HiveException(e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId, e.getMessage()); } } e = e.getCause(); } - return new HiveException(oe, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId); + return new HiveException(oe, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, sessionId, oe.getMessage()); } - @VisibleForTesting - String getMatchedString() { - return matchedString; - } - - private boolean matches(String input, String regex) { + private boolean matches(String input, String regex, StringBuilder matchedString) { if (!errorPatterns.containsKey(regex)) { LOG.warn("No error pattern found for regex: {}", regex); return false; @@ -235,7 +230,8 @@ private boolean matches(String input, String regex) { Matcher m = p.matcher(input); boolean result = m.find(); if (result && m.groupCount() == 1) { - this.matchedString = m.group(1); + // assume matchedString is empty + matchedString.append(m.group(1)); } return result; } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java index fe95ce0a85..5924b8ba45 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java @@ -183,8 +183,8 @@ public void testGetHiveException() throws Exception { "initial executor number 5 must between min executor number10 and max executor number 50"); // Other exceptions which defaults to SPARK_CREATE_CLIENT_ERROR - e = new Exception(); - checkHiveException(ss, e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR); + e = new Exception("Other exception"); + checkHiveException(ss, e, ErrorMsg.SPARK_CREATE_CLIENT_ERROR, "Other exception"); } private void checkHiveException(SparkSessionImpl ss, Throwable e, ErrorMsg expectedErrMsg) { @@ -196,7 +196,7 @@ private void checkHiveException(SparkSessionImpl ss, Throwable e, HiveException he = ss.getHiveException(e); assertEquals(expectedErrMsg, he.getCanonicalErrorMsg()); if (expectedMatchedStr != null) { - assertEquals(expectedMatchedStr, ss.getMatchedString()); + assertTrue(he.getMessage().contains(expectedMatchedStr)); } }