diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index f1e69d9..68f8cee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -616,6 +616,7 @@ public static void main(String[] args) throws IOException, HiveException { boolean noLog = false; String files = null; boolean localtask = false; + String hadoopAuthToken = null; try { for (int i = 0; i < args.length; i++) { if (args[i].equals("-plan")) { @@ -628,6 +629,8 @@ public static void main(String[] args) throws IOException, HiveException { files = args[++i]; } else if (args[i].equals("-localtask")) { localtask = true; + } else if (args[i].equals("-hadooptoken")) { + hadoopAuthToken = args[++i]; } } } catch (IndexOutOfBoundsException e) { @@ -649,6 +652,9 @@ public static void main(String[] args) throws IOException, HiveException { if (files != null) { conf.set("tmpfiles", files); } + if(hadoopAuthToken != null){ + conf.set("mapreduce.job.credentials.binary", hadoopAuthToken); + } boolean isSilent = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESESSIONSILENT); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java index f38ba94..9df90f3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java @@ -174,8 +174,6 @@ public int execute(DriverContext driverContext) { } } - LOG.info("Executing: " + cmdLine); - // Inherit Java system variables String hadoopOpts; StringBuilder sb = new StringBuilder(); @@ -231,14 +229,29 @@ public int execute(DriverContext driverContext) { MapRedTask.configureDebugVariablesForChildJVM(variables); } + + if(ShimLoader.getHadoopShims().isSecurityEnabled() && + conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS) == true + ){ + //If kerberos security is enabled, and HS2 doAs is enabled, + // then additional params need to be set so that the command is run as + // intended user + SecureCmdDoAs secureDoAs = new SecureCmdDoAs(conf); + cmdLine = secureDoAs.addArg(cmdLine); + secureDoAs.addEnv(variables); + } + env = new String[variables.size()]; int pos = 0; for (Map.Entry entry : variables.entrySet()) { String name = entry.getKey(); String value = entry.getValue(); env[pos++] = name + "=" + value; + LOG.debug("Setting env: " + env[pos-1]); } + LOG.info("Executing: " + cmdLine); + // Run ExecDriver in another JVM executor = Runtime.getRuntime().exec(cmdLine, env, new File(workDir)); diff --git a/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java b/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java index c9baa7f..a837e33 100644 --- a/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ b/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -594,6 +594,11 @@ public void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenS } @Override + public Path createDelegationTokenFile(Configuration conf) throws IOException { + throw new UnsupportedOperationException("Tokens are not supported in current hadoop version"); + } + + @Override public UserGroupInformation createRemoteUser(String userName, List groupNames) { return new UnixUserGroupInformation(userName, groupNames.toArray(new String[0])); } @@ -707,4 +712,11 @@ public long getDefaultBlockSize(FileSystem fs, Path path) { public short getDefaultReplication(FileSystem fs, Path path) { return fs.getDefaultReplication(); } + + @Override + public String getTokenFileLocEnvName() { + throw new UnsupportedOperationException( + "Kerberos not supported in current hadoop version"); + } + } diff --git a/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java b/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index 9edcafd..a649221 100644 --- a/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ b/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -19,6 +19,7 @@ import java.io.DataInput; import java.io.DataOutput; +import java.io.File; import java.io.IOException; import java.lang.reflect.Constructor; import java.net.URI; @@ -59,6 +60,7 @@ import org.apache.hadoop.mapred.lib.CombineFileInputFormat; import org.apache.hadoop.mapred.lib.CombineFileSplit; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -525,6 +527,26 @@ public void setTokenStr(UserGroupInformation ugi, String tokenStr, String tokenS } @Override + public Path createDelegationTokenFile(Configuration conf) throws IOException { + + //get delegation token for user + String uname = UserGroupInformation.getLoginUser().getShortUserName(); + FileSystem fs = FileSystem.get(conf); + Token fsToken = fs.getDelegationToken(uname); + + File t = File.createTempFile("hive_hadoop_delegation_token", null); + Path tokenPath = new Path(t.toURI()); + + //write credential with token to file + Credentials cred = new Credentials(); + cred.addToken(fsToken.getService(), fsToken); + cred.writeTokenStorageFile(tokenPath, conf); + + return tokenPath; + } + + + @Override public UserGroupInformation createProxyUser(String userName) throws IOException { return UserGroupInformation.createProxyUser( userName, UserGroupInformation.getLoginUser()); @@ -556,6 +578,11 @@ public void loginUserFromKeytab(String principal, String keytabFile) throws IOEx } @Override + public String getTokenFileLocEnvName() { + return UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION; + } + + @Override abstract public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception; @Override diff --git a/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java b/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java index 5a379df..1df6993 100644 --- a/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java +++ b/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.thrift; +import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION; + import java.io.IOException; import java.net.InetAddress; import java.net.Socket; @@ -40,8 +42,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Client; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport; import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; @@ -64,8 +64,6 @@ import org.apache.thrift.transport.TTransportException; import org.apache.thrift.transport.TTransportFactory; -import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION; - /** * Functions that bridge Thrift's SASL transports to Hadoop's * SASL callback handlers and authentication classes. @@ -359,7 +357,9 @@ public String getDelegationToken(final String owner, final String renewer) throws IOException, InterruptedException { if (!authenticationMethod.get().equals(AuthenticationMethod.KERBEROS)) { throw new AuthorizationException( - "Delegation Token can be issued only with kerberos authentication"); + "Delegation Token can be issued only with kerberos authentication. " + + "Current AuthenticationMethod: " + authenticationMethod.get() + ); } //if the user asking the token is same as the 'owner' then don't do //any proxy authorization checks. For cases like oozie, where it gets @@ -388,7 +388,9 @@ public String run() throws IOException { public long renewDelegationToken(String tokenStrForm) throws IOException { if (!authenticationMethod.get().equals(AuthenticationMethod.KERBEROS)) { throw new AuthorizationException( - "Delegation Token can be issued only with kerberos authentication"); + "Delegation Token can be issued only with kerberos authentication. " + + "Current AuthenticationMethod: " + authenticationMethod.get() + ); } return secretManager.renewDelegationToken(tokenStrForm); } @@ -430,7 +432,7 @@ protected synchronized String initialValue() { public String getRemoteUser() { return remoteUser.get(); } - + /** CallbackHandler for SASL DIGEST-MD5 mechanism */ // This code is pretty much completely based on Hadoop's // SaslRpcServer.SaslDigestCallbackHandler - the only reason we could not diff --git a/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java index b0f5077..6a7da82 100644 --- a/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ b/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -197,16 +197,18 @@ public URI getHarUri(URI original, URI base, URI originalBase) */ public String unquoteHtmlChars(String item); + + + public void closeAllForUGI(UserGroupInformation ugi); + /** * Get the UGI that the given job configuration will run as. * * In secure versions of Hadoop, this simply returns the current * access control context's user, ignoring the configuration. */ - - public void closeAllForUGI(UserGroupInformation ugi); - public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException, IOException; + /** * Used by metastore server to perform requested rpc in client context. * @param @@ -219,6 +221,26 @@ public URI getHarUri(URI original, URI base, URI originalBase) IOException, InterruptedException; /** + * Once a delegation token is stored in a file, the location is specified + * for a child process that runs hadoop operations, using an environment + * variable . + * @return Return the name of environment variable used by hadoop to find + * location of token file + */ + public String getTokenFileLocEnvName(); + + + /** + * Get delegation token from filesystem and write the token along with + * metastore tokens into a file + * @param conf + * @return Path of the file with token credential + * @throws IOException + */ + public Path createDelegationTokenFile(final Configuration conf) throws IOException; + + + /** * Used by metastore server to creates UGI object for a remote user. * @param userName remote User Name * @param groupNames group names associated with remote user name