From c2c5567540e85e181cae0baab097560591743831 Mon Sep 17 00:00:00 2001 From: windpiger Date: Thu, 14 Jun 2018 17:06:50 +0800 Subject: [PATCH] remote spark driver use ip to connect Hive Client --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 2 ++ .../org/apache/hive/spark/client/rpc/RpcConfiguration.java | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 8a45b9cc9a..7096908eae 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4238,6 +4238,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Default is empty, which means the address will be determined in the same way as for hive.server2.thrift.bind.host." + "This is only necessary if the host has multiple network addresses and if a different network address other than " + "hive.server2.thrift.bind.host is to be used."), + SPARK_RPC_SERVER_ADDRESS_USE_IP("hive.spark.client.rpc.server.address.use.ip", false, "determine whether to use ip for " + + "hive.spark.client.rpc.server.address, if set to true, remote Spark driver will use ip(not hostname) to connect Hive Client"), SPARK_RPC_SERVER_PORT("hive.spark.client.rpc.server.port", "", "A list of port ranges which can be used by RPC server " + "with the format of 49152-49222,49228 and a random one is selected from the list. Default is empty, which randomly " + "selects one port from all available ones."), diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/rpc/RpcConfiguration.java b/spark-client/src/main/java/org/apache/hive/spark/client/rpc/RpcConfiguration.java index bd3a7a7321..51204f8f5b 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/rpc/RpcConfiguration.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/rpc/RpcConfiguration.java @@ -53,7 +53,8 @@ HiveConf.ConfVars.SPARK_RPC_MAX_MESSAGE_SIZE.varname, HiveConf.ConfVars.SPARK_RPC_MAX_THREADS.varname, HiveConf.ConfVars.SPARK_RPC_SECRET_RANDOM_BITS.varname, - HiveConf.ConfVars.SPARK_RPC_SERVER_ADDRESS.varname + HiveConf.ConfVars.SPARK_RPC_SERVER_ADDRESS.varname, + HiveConf.ConfVars.SPARK_RPC_SERVER_ADDRESS_USE_IP.varname ); public static final ImmutableSet HIVE_SPARK_TIME_CONFIGS = ImmutableSet.of( HiveConf.ConfVars.SPARK_CLIENT_FUTURE_TIMEOUT.varname, @@ -116,7 +117,13 @@ String getServerAddress() throws IOException { hiveHost = config.get(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST.varname); } } - return ServerUtils.getHostAddress(hiveHost).getHostName(); + + String isUseIp = config.get(HiveConf.ConfVars.SPARK_RPC_SERVER_ADDRESS_USE_IP.varname); + if(isUseIp != null && Boolean.valueOf(isUseIp)) { + return ServerUtils.getHostAddress(hiveHost).getHostAddress(); + } else { + return ServerUtils.getHostAddress(hiveHost).getHostName(); + } } /** -- 2.14.2