From 086e357d0bd224c091e3d56b0ffd7085b11b77e3 Mon Sep 17 00:00:00 2001 From: Sam An Date: Wed, 1 May 2019 13:36:43 -0700 Subject: [PATCH] HIVE-21672 HiveServer2 needs to support sidecar's 'ldap.xml' file. (Sam An, reviewed by Daniel Dai) --- .../org/apache/hadoop/hive/conf/HiveConf.java | 22 ++++++- .../apache/hadoop/hive/conf/TestHiveConf.java | 58 ++++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 0c2bd1e5ad..0bfebbb2ed 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -86,6 +86,7 @@ private static URL hiveSiteURL = null; private static URL hivemetastoreSiteUrl = null; private static URL hiveServer2SiteUrl = null; + private static URL hiveSideCarURL = null; private static byte[] confVarByteArray = null; @@ -4703,7 +4704,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "This parameter enables a number of optimizations when running on blobstores:\n" + "(1) If hive.blobstore.use.blobstore.as.scratchdir is false, force the last Hive job to write to the blobstore.\n" + "This is a performance optimization that forces the final FileSinkOperator to write to the blobstore.\n" + - "See HIVE-15121 for details."); + "See HIVE-15121 for details."), + + HIVE_ADDITIONAL_CONFIG_FILES("hive.additional.config.files", "", + "The names of additional config files, such as ldap-site.xml,spark-site.xml, etc in comma separated list." ); public final String varname; public final String altName; @@ -5474,6 +5478,18 @@ private void initialize(Class cls) { addResource(hiveServer2SiteUrl); } + String val = this.getVar(HiveConf.ConfVars.HIVE_ADDITIONAL_CONFIG_FILES); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + if ( val != null && !val.isEmpty()) { + String[] configFiles = val.split(","); + for (String config : configFiles) { + URL configURL = findConfigFile(classLoader, config, true); + if (configURL != null) { + addResource(configURL); + } + } + } // Overlay the values of any system properties and manual overrides applySystemProperties(); @@ -5855,6 +5871,10 @@ public static URL getHiveServer2SiteLocation() { return hiveServer2SiteUrl; } + public static URL getHiveSideCarURL(){ + return hiveSideCarURL; + } + /** * @return the user name set in hadoop.job.ugi param or the current user from System * @throws IOException diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java index 780a708dab..a3151a3f27 100644 --- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java +++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java @@ -18,15 +18,18 @@ package org.apache.hadoop.hive.conf; import com.google.common.collect.Lists; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.util.Shell; import org.apache.hive.common.util.HiveTestUtils; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; -import java.io.UnsupportedEncodingException; +import java.io.*; +import java.net.URL; import java.net.URLEncoder; import java.util.ArrayList; import java.util.concurrent.TimeUnit; @@ -189,4 +192,57 @@ public void testEncodingDecoding() throws UnsupportedEncodingException { Assert.assertEquals(URLEncoder.encode(query, "UTF-8"), conf.get(ConfVars.HIVEQUERYSTRING.varname)); Assert.assertEquals(query, conf.getQueryString()); } + + @Test + public void testAdditionalConfigFiles() throws Exception{ + URL url = ClassLoader.getSystemResource("hive-site.xml"); + File fileHiveSite = new File(url.getPath()); + + String parFolder = fileHiveSite.getParent(); + //back up hive-site.xml + String bakHiveSiteFileName = parFolder + "/hive-site-bak.xml"; + File fileBakHiveSite = new File(bakHiveSiteFileName); + FileUtils.copyFile(fileHiveSite, fileBakHiveSite); + + String content = FileUtils.readFileToString(fileHiveSite); + content = content.substring(0, content.lastIndexOf("")); + + String testHiveSiteString = content + "\n" + + " HIVE_SERVER2_PLAIN_LDAP_DOMAIN\n" + + " a.com\n" + + "\n" + + "\n" + + " \n" + + " hive.additional.config.files\n" + + " ldap-site.xml,other.xml\n" + + " additional config dir for Hive to load\n" + + " \n" + + "\n" + + ""; + + FileUtils.writeStringToFile(fileHiveSite, testHiveSiteString); + + String testLdapString = "\n" + + "\n" + + "\n" + + " \n" + + " hive.server2.authentication.ldap.Domain\n" + + " b.com\n" + + "\n" + + "\n" + + ""; + + + String newFileName = parFolder+"/ldap-site.xml"; + File f2 = new File(newFileName); + FileUtils.writeStringToFile(f2, testLdapString); + + HiveConf conf = new HiveConf(); + String val = conf.getVar(ConfVars.HIVE_SERVER2_PLAIN_LDAP_DOMAIN); + Assert.assertEquals("b.com", val); + //restore and clean up + FileUtils.copyFile(fileBakHiveSite, fileHiveSite); + f2.delete(); + fileBakHiveSite.delete(); + } } -- 2.20.1 From 44d1b0cb061df0db4eaef2afb1929bf4b2db2e17 Mon Sep 17 00:00:00 2001 From: Sam An Date: Wed, 1 May 2019 15:57:07 -0700 Subject: [PATCH] Correct check style errors --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 5 +++-- .../src/test/org/apache/hadoop/hive/conf/TestHiveConf.java | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 0bfebbb2ed..a3737faa6b 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4707,7 +4707,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "See HIVE-15121 for details."), HIVE_ADDITIONAL_CONFIG_FILES("hive.additional.config.files", "", - "The names of additional config files, such as ldap-site.xml,spark-site.xml, etc in comma separated list." ); + "The names of additional config files, such as ldap-site.xml," + + "spark-site.xml, etc in comma separated list."); public final String varname; public final String altName; @@ -5481,7 +5482,7 @@ private void initialize(Class cls) { String val = this.getVar(HiveConf.ConfVars.HIVE_ADDITIONAL_CONFIG_FILES); ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); - if ( val != null && !val.isEmpty()) { + if (val != null && !val.isEmpty()) { String[] configFiles = val.split(","); for (String config : configFiles) { URL configURL = findConfigFile(classLoader, config, true); diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java index a3151a3f27..bf9dee7825 100644 --- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java +++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java @@ -22,13 +22,12 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.util.Shell; import org.apache.hive.common.util.HiveTestUtils; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; -import java.io.*; +import java.io.File; +import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLEncoder; import java.util.ArrayList; -- 2.20.1 From 10d075a4579fa4cad1588b2714f1a1e64d5b0d7f Mon Sep 17 00:00:00 2001 From: Sam An Date: Fri, 3 May 2019 09:14:39 -0700 Subject: [PATCH] Remove unused variable and method --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index a3737faa6b..049e83713e 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -86,7 +86,6 @@ private static URL hiveSiteURL = null; private static URL hivemetastoreSiteUrl = null; private static URL hiveServer2SiteUrl = null; - private static URL hiveSideCarURL = null; private static byte[] confVarByteArray = null; @@ -5872,10 +5871,6 @@ public static URL getHiveServer2SiteLocation() { return hiveServer2SiteUrl; } - public static URL getHiveSideCarURL(){ - return hiveSideCarURL; - } - /** * @return the user name set in hadoop.job.ugi param or the current user from System * @throws IOException -- 2.20.1