From abf1ae91be93a26729c34b79866d0a6c4883f264 Mon Sep 17 00:00:00 2001 From: Sam An Date: Fri, 20 Mar 2020 14:41:35 -0700 Subject: [PATCH] HIVE-23062 Hive to check Yarn RM URL in TLS and Yarn HA mode for custom Tez queue --- .../hive/ql/exec/tez/YarnQueueHelper.java | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java index b762e68db8b..800e9fe4a82 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/YarnQueueHelper.java @@ -23,6 +23,8 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hive.conf.HiveConf; @@ -30,6 +32,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.http.HttpStatus; import org.json.JSONException; import org.json.JSONObject; @@ -44,11 +47,47 @@ private final String[] rmNodes; private int lastKnownGoodUrl; + private boolean sslForYarn; + private boolean isHA; + private static String webapp_conf_key = YarnConfiguration.RM_WEBAPP_ADDRESS; + private static String webapp_ssl_conf_key = YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS; + private static String yarn_HA_enabled = YarnConfiguration.RM_HA_ENABLED; + private static String yarn_HA_rmids = YarnConfiguration.RM_HA_IDS; public YarnQueueHelper(HiveConf conf) { - rmNodes = conf.getTrimmedStrings("yarn.resourcemanager.webapp.address"); - Preconditions.checkArgument((rmNodes != null && rmNodes.length > 0), - "yarn.resourcemanager.webapp.address must be set to enable queue access checks"); + ArrayList nodeList = new ArrayList<>(); + sslForYarn = YarnConfiguration.useHttps(conf); + isHA = conf.getBoolean(yarn_HA_enabled, false); + LOG.info(String.format("HS2 is using SSL: %s", sslForYarn)); + LOG.info(String.format("Yarn HA is enabled: %s", isHA)); + + if (isHA) { + String[] rmids = conf.getStrings(yarn_HA_rmids); + if (sslForYarn == true) { + for (String rmid : rmids) { + nodeList.addAll(Arrays.asList(conf.getTrimmedStrings(webapp_ssl_conf_key + "."+rmid))); + } + Preconditions.checkArgument(nodeList.size() > 0, + "yarn.resourcemanager.ha.rm-ids must be set to enable queue access checks in Yarn HA mode"); + }else{ + for (String rmid : rmids) { + nodeList.addAll(Arrays.asList(conf.getTrimmedStrings(webapp_conf_key + "."+rmid))); + Preconditions.checkArgument(nodeList.size() > 0, + "yarn.resourcemanager.ha.rm-ids must be set to enable queue access checks in Yarn HA mode"); + } + } + rmNodes = nodeList.toArray(new String[nodeList.size()]); + }else { + if (sslForYarn == true) { + rmNodes = conf.getTrimmedStrings(webapp_ssl_conf_key); + Preconditions.checkArgument((rmNodes != null && rmNodes.length > 0), + "yarn.resourcemanager.webapp.https.address must be set to enable queue access checks using TLS"); + } else { + rmNodes = conf.getTrimmedStrings(webapp_conf_key); + Preconditions.checkArgument((rmNodes != null && rmNodes.length > 0), + "yarn.resourcemanager.webapp.address must be set to enable queue access checks"); + } + } lastKnownGoodUrl = 0; } @@ -62,8 +101,14 @@ public void checkQueueAccess( String node = rmNodes[urlIx]; String error = null; boolean isCallOk = false; + String urlToCheck; + if (sslForYarn){ + urlToCheck = "https://" + node + urlSuffix; + }else{ + urlToCheck = "http://" + node + urlSuffix; + } try { - error = checkQueueAccessFromSingleRm("http://" + node + urlSuffix); + error = checkQueueAccessFromSingleRm(urlToCheck); isCallOk = true; } catch (Exception ex) { LOG.warn("Cannot check queue access against RM " + node, ex); -- 2.23.0