From 6220588c92bde9ea753c46d3ca53d71c5802040f Mon Sep 17 00:00:00 2001 From: wyj Date: Tue, 25 Jul 2017 22:33:56 -0700 Subject: [PATCH 1/3] KYLIN-2755:Kylin support hive and hbase authenticated with Kerberos --- .../org/apache/kylin/common/KylinConfigBase.java | 40 ++++++++++++++++++++++ .../apache/kylin/common/util/HiveCmdBuilder.java | 22 ++++++++++++ .../kylin/source/hive/BeelineHiveClient.java | 2 ++ .../kylin/storage/hbase/HBaseConnection.java | 21 ++++++++++++ .../kylin/storage/hbase/steps/CubeHFileJob.java | 14 ++++---- 5 files changed, 91 insertions(+), 8 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index f9c3adb..a273bc5 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -585,6 +585,26 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.source.hive.client", "cli"); } + public boolean getHivekerberized() { + String hiveAuthentication = getOptional("kylin.hive.authentication", ""); + if (!(StringUtils.isEmpty(hiveAuthentication) + || "kerberos".equals(hiveAuthentication))) + throw new IllegalArgumentException("to use hive kerberos, pls set 'kylin.hive.authentication=kerberos' in kylin.properties"); + + if("kerberos".equals(hiveAuthentication)) + return true; + else + return false; + } + + public String getHiveKerberosPrincipal() { + return getOptional("kylin.hive.kerberos.principal", ""); + } + + public String getHiveKerberosKeytab() { + return getOptional("kylin.hive.kerberos.keytab", ""); + } + public String getHiveBeelineParams() { return getOptional("kylin.source.hive.beeline-params", ""); } @@ -633,6 +653,26 @@ abstract public class KylinConfigBase implements Serializable { // STORAGE.HBASE // ============================================================================ + public boolean getHbasekerberized() { + String hbaseAuthentication = getOptional("kylin.hbase.authentication", ""); + if (!(StringUtils.isEmpty(hbaseAuthentication) + || "kerberos".equals(hbaseAuthentication))) + throw new IllegalArgumentException("to use hbase kerberos, pls set 'kylin.hbase.authentication=kerberos' in kylin.properties"); + + if("kerberos".equals(hbaseAuthentication)) + return true; + else + return false; + } + + public String getHBaseKerberosPrincipal() { + return getOptional("kylin.hbase.kerberos.principal", ""); + } + + public String getHBaseKerberosKeytab() { + return getOptional("kylin.hbase.kerberos.keytab", ""); + } + public Map getStorageEngines() { Map r = Maps.newLinkedHashMap(); // ref constants in IStorageAware diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index 2f6b9a0..b30f05b 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -33,6 +33,8 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.kylin.common.KylinConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +61,7 @@ public class HiveCmdBuilder { kylinConfig = KylinConfig.getInstanceFromEnv(); clientMode = HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase()); loadHiveConfiguration(); + hiveKerberize(); } public String build() { @@ -198,4 +201,23 @@ public class HiveCmdBuilder { throw new RuntimeException("Failed to parse hive conf file ", e); } } + + public static void hiveKerberize() { + boolean hivekerberized = KylinConfig.getInstanceFromEnv().getHivekerberized(); + + if (hivekerberized) { + Configuration conf = new Configuration(); + conf.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(conf); + + try { + String kerberosPrincipal = KylinConfig.getInstanceFromEnv().getHiveKerberosPrincipal(); + String kerberosKeytabLocation = KylinConfig.getInstanceFromEnv().getHiveKerberosKeytab(); + UserGroupInformation.loginUserFromKeytab(kerberosPrincipal, kerberosKeytabLocation); + }catch (IOException e){ + throw new RuntimeException(e); + } + } + + } } \ No newline at end of file diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java index ee693c5..7bd2a68 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java @@ -33,6 +33,7 @@ import org.apache.kylin.common.util.DBUtils; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.kylin.common.util.HiveCmdBuilder; public class BeelineHiveClient implements IHiveClient { @@ -62,6 +63,7 @@ public class BeelineHiveClient implements IHiveClient { private void init(String url, String username, String password) { try { + HiveCmdBuilder.hiveKerberize(); Class.forName("org.apache.hive.jdbc.HiveDriver"); cnct = DriverManager.getConnection(url, username, password); stmt = cnct.createStatement(); diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java index 6580107..a2aacf4 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.lock.DistributedLock; @@ -134,6 +135,7 @@ public class HBaseConnection { } public static Configuration getCurrentHBaseConfiguration() { + hbaseKerberize(); if (configThreadLocal.get() == null) { StorageURL storageUrl = KylinConfig.getInstanceFromEnv().getStorageUrl(); configThreadLocal.set(newHBaseConfiguration(storageUrl)); @@ -141,6 +143,25 @@ public class HBaseConnection { return configThreadLocal.get(); } + public static void hbaseKerberize() { + //login user from key tab + boolean hbasekerberized = KylinConfig.getInstanceFromEnv().getHbasekerberized(); + if(hbasekerberized){ + Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()); + conf.set("hadoop.security.authentication", "Kerberos"); + UserGroupInformation.setConfiguration(conf); + + try { + String kerberosPrincipal = KylinConfig.getInstanceFromEnv().getHBaseKerberosPrincipal(); + String kerberosKeytabLocation = KylinConfig.getInstanceFromEnv().getHBaseKerberosKeytab(); + UserGroupInformation.loginUserFromKeytab(kerberosPrincipal, kerberosKeytabLocation); + } catch (IOException e) { + logger.error("can not login user from hbase key tab ", e); + throw new RuntimeException(e); + } + } + } + private static Configuration newHBaseConfiguration(StorageURL url) { // using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath if (!"hbase".equals(url.getScheme())) diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java index 1a624c4..40c6404 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java @@ -24,7 +24,6 @@ import org.apache.commons.cli.Options; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer; @@ -73,11 +72,11 @@ public class CubeHFileJob extends AbstractHadoopJob { CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeInstance cube = cubeMgr.getCube(cubeName); - job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME)); + + Configuration configuration = HBaseConnection.getCurrentHBaseConfiguration(); + job = Job.getInstance(configuration, getOptionValue(OPTION_JOB_NAME)); setJobClasspath(job, cube.getConfig()); - // For separate HBase cluster, note the output is a qualified HDFS path if "kylin.storage.hbase.cluster-fs" is configured, ref HBaseMRSteps.getHFilePath() - HBaseConnection.addHBaseClusterNNHAConfiguration(job.getConfiguration()); addInputDirs(getOptionValue(OPTION_INPUT_PATH), job); FileOutputFormat.setOutputPath(job, output); @@ -91,15 +90,14 @@ public class CubeHFileJob extends AbstractHadoopJob { // add metadata to distributed cache attachCubeMetadata(cube, job.getConfiguration()); - Configuration hbaseConf = HBaseConfiguration.create(getConf()); - HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME).toUpperCase()); + HTable htable = new HTable(configuration, getOptionValue(OPTION_HTABLE_NAME).toUpperCase()); // Automatic config ! HFileOutputFormat.configureIncrementalLoad(job, htable); - reconfigurePartitions(hbaseConf, partitionFilePath); + reconfigurePartitions(configuration, partitionFilePath); // set block replication to 3 for hfiles - hbaseConf.set(DFSConfigKeys.DFS_REPLICATION_KEY, "3"); + configuration.set(DFSConfigKeys.DFS_REPLICATION_KEY, "3"); this.deletePath(job.getConfiguration(), output); -- 1.9.1 From dd8c6c2a9ea5fc89e800fe36076118b9488c5f28 Mon Sep 17 00:00:00 2001 From: wyj Date: Tue, 25 Jul 2017 23:15:27 -0700 Subject: [PATCH 2/3] KYLIN-2755:Kylin support hive and hbase authenticated with Kerberos:delete space line --- .../src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index b30f05b..a85f852 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -218,6 +218,5 @@ public class HiveCmdBuilder { throw new RuntimeException(e); } } - } } \ No newline at end of file -- 1.9.1 From 2a3d73b86d7f7128355fff7bd3b31a6f48ad7fa9 Mon Sep 17 00:00:00 2001 From: wyj Date: Tue, 25 Jul 2017 23:50:25 -0700 Subject: [PATCH 3/3] KYLIN-2755:Kylin support hive and hbase authenticated with Kerberos:format with Kerberos --- .../src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java | 2 +- .../src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index a85f852..c93b295 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -214,7 +214,7 @@ public class HiveCmdBuilder { String kerberosPrincipal = KylinConfig.getInstanceFromEnv().getHiveKerberosPrincipal(); String kerberosKeytabLocation = KylinConfig.getInstanceFromEnv().getHiveKerberosKeytab(); UserGroupInformation.loginUserFromKeytab(kerberosPrincipal, kerberosKeytabLocation); - }catch (IOException e){ + } catch (IOException e) { throw new RuntimeException(e); } } diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java index a2aacf4..c6523f2 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java @@ -146,7 +146,7 @@ public class HBaseConnection { public static void hbaseKerberize() { //login user from key tab boolean hbasekerberized = KylinConfig.getInstanceFromEnv().getHbasekerberized(); - if(hbasekerberized){ + if(hbasekerberized) { Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()); conf.set("hadoop.security.authentication", "Kerberos"); UserGroupInformation.setConfiguration(conf); -- 1.9.1