diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index bd25bc7..5374859 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -91,6 +91,7 @@ private static final Map metaConfs = new HashMap(); private final List restrictList = new ArrayList(); private final Set hiddenSet = new HashSet(); + private final List rscList = new ArrayList<>(); private Pattern modWhiteListPattern = null; private volatile boolean isSparkConfigUpdated = false; @@ -3569,6 +3570,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "hive.spark.client.secret.bits," + "hive.spark.client.rpc.server.address," + "hive.spark.client.rpc.server.port," + + "hive.spark.client.rpc.sasl.mechanisms," + "bonecp.,"+ "hive.druid.broker.address.default,"+ "hive.druid.coordinator.address.default,"+ @@ -3589,6 +3591,12 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "hive.added.files.path,hive.added.jars.path,hive.added.archives.path", "Comma separated list of variables which are used internally and should not be configurable."), + HIVE_SPARK_RSC_CONF_LIST("hive.spark.rsc.conf.list", + SPARK_OPTIMIZE_SHUFFLE_SERDE.varname + "," + + SPARK_CLIENT_FUTURE_TIMEOUT.varname, + "Comma separated list of variables which are related to remote spark context. " + + "Changing these variables will result in re-creating the spark session."), + HIVE_QUERY_TIMEOUT_SECONDS("hive.query.timeout.seconds", "0s", new TimeValidator(TimeUnit.SECONDS), "Timeout for Running Query in seconds. A nonpositive value means infinite. " + @@ -3916,7 +3924,7 @@ private boolean isSparkRelatedConfig(String name) { if (sparkMaster != null && sparkMaster.startsWith("yarn")) { result = true; } - } else if (name.startsWith("hive.spark")) { // Remote Spark Context property. + } else if (rscList.stream().anyMatch(rscVar -> rscVar.equals(name))) { // Remote Spark Context property. result = true; } else if (name.equals("mapreduce.job.queuename")) { // a special property starting with mapreduce that we would also like to effect if it changes @@ -4398,6 +4406,7 @@ private void initialize(Class cls) { setupRestrictList(); hiddenSet.clear(); hiddenSet.addAll(HiveConfUtil.getHiddenSet(this)); + setupRSCList(); } /** @@ -4788,6 +4797,17 @@ private void setupRestrictList() { restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname); restrictList.add(ConfVars.HIVE_CONF_HIDDEN_LIST.varname); restrictList.add(ConfVars.HIVE_CONF_INTERNAL_VARIABLE_LIST.varname); + restrictList.add(ConfVars.HIVE_SPARK_RSC_CONF_LIST.varname); + } + + private void setupRSCList() { + rscList.clear(); + String vars = this.getVar(ConfVars.HIVE_SPARK_RSC_CONF_LIST); + if (vars != null) { + for (String var : vars.split(",")) { + rscList.add(var.trim()); + } + } } /**