From f431535c79c21ae8a82de1359a25d69ce1c67cf7 Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Mon, 22 Sep 2014 15:22:54 -0700 Subject: [PATCH] Starting to work on user changes. --- .../client/coprocessor/SecureBulkLoadClient.java | 4 +- .../org/apache/hadoop/hbase/ipc/RpcClient.java | 17 +- .../hadoop/hbase/security/HBaseSaslRpcClient.java | 9 +- .../org/apache/hadoop/hbase/security/User.java | 381 +++++++++++++++++++++ .../apache/hadoop/hbase/security/UserProvider.java | 118 +++++++ .../token/AuthenticationTokenSelector.java | 14 +- .../hadoop/hbase/security/token/HBToken.java | 153 +++++++++ .../hbase/security/token/ReflectionHolder.java | 106 ++++++ .../hadoop/hbase/io/compress/Compression.java | 13 +- .../org/apache/hadoop/hbase/security/User.java | 367 -------------------- .../apache/hadoop/hbase/security/UserProvider.java | 118 ------- .../hadoop/hbase/util/CoprocessorClassLoader.java | 3 +- .../hadoop/hbase/util/DynamicClassLoader.java | 2 +- .../hadoop/hbase/mapred/TableMapReduceUtil.java | 9 +- .../hadoop/hbase/mapreduce/TableMapReduceUtil.java | 24 +- .../security/access/SecureBulkLoadEndpoint.java | 17 +- .../hbase/security/token/FsDelegationToken.java | 11 +- .../hbase/security/TestHBaseSaslRpcClient.java | 20 +- pom.xml | 4 +- 19 files changed, 846 insertions(+), 544 deletions(-) create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/security/User.java create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/HBToken.java create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/ReflectionHolder.java delete mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/security/User.java delete mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/SecureBulkLoadClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/SecureBulkLoadClient.java index 90564e4..7c78039 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/SecureBulkLoadClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/SecureBulkLoadClient.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.HConstants.EMPTY_START_ROW; import static org.apache.hadoop.hbase.HConstants.LAST_ROW; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.security.token.HBToken; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -35,7 +36,6 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos; import org.apache.hadoop.hbase.security.SecureBulkLoadUtil; import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.security.token.Token; import java.io.IOException; import java.util.ArrayList; @@ -121,7 +121,7 @@ public class SecureBulkLoadClient { } public boolean bulkLoadHFiles(final List> familyPaths, - final Token userToken, + final HBToken userToken, final String bulkToken, final byte[] startRow) throws IOException { // we never want to send a batch of HFiles to all regions, thus cannot call diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java index 1bfd9a6..28ec0c9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.security.SecurityInfo; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector; +import org.apache.hadoop.hbase.security.token.HBToken; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.ExceptionUtil; import org.apache.hadoop.hbase.util.Pair; @@ -67,9 +68,6 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.TokenSelector; import org.htrace.Span; import org.htrace.Trace; import org.htrace.TraceScope; @@ -336,8 +334,8 @@ public class RpcClient { } protected final static Map> tokenHandlers = - new HashMap>(); + AuthenticationTokenSelector> tokenHandlers = + new HashMap(); static { tokenHandlers.put(AuthenticationProtos.TokenIdentifier.Kind.HBASE_AUTH_TOKEN, new AuthenticationTokenSelector()); @@ -385,7 +383,7 @@ public class RpcClient { private String serverPrincipal; // server's krb5 principal name private AuthMethod authMethod; // authentication method private boolean useSasl; - private Token token; + private HBToken token; private HBaseSaslRpcClient saslRpcClient; private int reloginMaxBackoff; // max pause before relogin on sasl failure private final Codec codec; @@ -523,17 +521,18 @@ public class RpcClient { this.codec = codec; this.compressor = compressor; - UserGroupInformation ticket = remoteId.getTicket().getUGI(); + User user = remoteId.getTicket(); + UserGroupInformation ticket = user.getUGI(); SecurityInfo securityInfo = SecurityInfo.getInfo(remoteId.getServiceName()); this.useSasl = userProvider.isHBaseSecurityEnabled(); if (useSasl && securityInfo != null) { AuthenticationProtos.TokenIdentifier.Kind tokenKind = securityInfo.getTokenKind(); if (tokenKind != null) { - TokenSelector tokenSelector = + AuthenticationTokenSelector tokenSelector = tokenHandlers.get(tokenKind); if (tokenSelector != null) { token = tokenSelector.selectToken(new Text(clusterId), - ticket.getTokens()); + user.getTokens()); } else if (LOG.isDebugEnabled()) { LOG.debug("No token selector found for type "+tokenKind); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java index 74a9f35..a27eef8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java @@ -21,12 +21,11 @@ package org.apache.hadoop.hbase.security; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.security.token.HBToken; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.SaslInputStream; import org.apache.hadoop.security.SaslOutputStream; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import javax.security.auth.callback.Callback; import javax.security.auth.callback.CallbackHandler; @@ -73,7 +72,7 @@ public class HBaseSaslRpcClient { * @throws IOException */ public HBaseSaslRpcClient(AuthMethod method, - Token token, String serverPrincipal, boolean fallbackAllowed) + HBToken token, String serverPrincipal, boolean fallbackAllowed) throws IOException { this(method, token, serverPrincipal, fallbackAllowed, "authentication"); } @@ -93,7 +92,7 @@ public class HBaseSaslRpcClient { * @throws IOException */ public HBaseSaslRpcClient(AuthMethod method, - Token token, String serverPrincipal, boolean fallbackAllowed, + HBToken token, String serverPrincipal, boolean fallbackAllowed, String rpcProtection) throws IOException { this.fallbackAllowed = fallbackAllowed; SaslUtil.initSaslProperties(rpcProtection); @@ -282,7 +281,7 @@ public class HBaseSaslRpcClient { private final String userName; private final char[] userPassword; - public SaslClientCallbackHandler(Token token) { + public SaslClientCallbackHandler(HBToken token) { this.userName = SaslUtil.encodeIdentifier(token.getIdentifier()); this.userPassword = SaslUtil.encodePassword(token.getPassword()); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/User.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/User.java new file mode 100644 index 0000000..a844030 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/User.java @@ -0,0 +1,381 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.security; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.UndeclaredThrowableException; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.security.token.HBToken; +import org.apache.hadoop.hbase.security.token.ReflectionHolder; +import org.apache.hadoop.hbase.util.Methods; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.security.UserGroupInformation; +/** + * Wrapper to abstract out usage of user and group information in HBase. + * + *

+ * This class provides a common interface for interacting with user and group + * information across changing APIs in different versions of Hadoop. It only + * provides access to the common set of functionality in + * {@link org.apache.hadoop.security.UserGroupInformation} currently needed by + * HBase, but can be extended as needs change. + *

+ */ +@InterfaceAudience.Private +public abstract class User { + public static final String HBASE_SECURITY_CONF_KEY = + "hbase.security.authentication"; + + protected UserGroupInformation ugi; + + public UserGroupInformation getUGI() { + return ugi; + } + + /** + * Returns the full user name. For Kerberos principals this will include + * the host and realm portions of the principal name. + * @return User full name. + */ + public String getName() { + return ugi.getUserName(); + } + + /** + * Returns the list of groups of which this user is a member. On secure + * Hadoop this returns the group information for the user as resolved on the + * server. For 0.20 based Hadoop, the group names are passed from the client. + */ + public String[] getGroupNames() { + return ugi.getGroupNames(); + } + + /** + * Returns the shortened version of the user name -- the portion that maps + * to an operating system user name. + * @return Short name + */ + public abstract String getShortName(); + + /** + * Executes the given action within the context of this user. + */ + public abstract T runAs(PrivilegedAction action); + + /** + * Executes the given action within the context of this user. + */ + public abstract T runAs(PrivilegedExceptionAction action) + throws IOException, InterruptedException; + + /** + * Requests an authentication token for this user and stores it in the + * user's credentials. + * + * @throws IOException + */ + public abstract void obtainAuthTokenForJob(Configuration conf, Job job) + throws IOException, InterruptedException; + + /** + * Requests an authentication token for this user and stores it in the + * user's credentials. + * + * @throws IOException + */ + public abstract void obtainAuthTokenForJob(JobConf job) + throws IOException, InterruptedException; + + + public Collection getTokens() { + return HBToken.getTokensFromUGI(ugi); + } + + /** + * Returns the Token of the specified kind associated with this user, + * or null if the Token is not present. + * + * @param kind the kind of token + * @param service service on which the token is supposed to be used + * @return the token of the specified kind. + */ + public HBToken getToken(String kind, String service) throws IOException { + for (HBToken token: getTokens()) { + if (token.getKind().toString().equals(kind) && + (service != null && token.getService().toString().equals(service))) + { + return token; + } + } + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return ugi.equals(((User) o).ugi); + } + + @Override + public int hashCode() { + return ugi.hashCode(); + } + + @Override + public String toString() { + return ugi.toString(); + } + + /** + * Returns the {@code User} instance within current execution context. + */ + public static User getCurrent() throws IOException { + User user = new SecureHadoopUser(); + if (user.getUGI() == null) { + return null; + } + return user; + } + + /** + * Executes the given action as the login user + * @param action + * @return the result of the action + * @throws IOException + * @throws InterruptedException + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + public static T runAsLoginUser(PrivilegedExceptionAction action) throws IOException { + try { + Class c = Class.forName("org.apache.hadoop.security.SecurityUtil"); + Class [] types = new Class[]{PrivilegedExceptionAction.class}; + Object[] args = new Object[]{action}; + return (T) Methods.call(c, null, "doAsLoginUser", types, args); + } catch (Throwable e) { + throw new IOException(e); + } + } + + /** + * Wraps an underlying {@code UserGroupInformation} instance. + * @param ugi The base Hadoop user + * @return User + */ + public static User create(UserGroupInformation ugi) { + if (ugi == null) { + return null; + } + return new SecureHadoopUser(ugi); + } + + /** + * Generates a new {@code User} instance specifically for use in test code. + * @param name the full username + * @param groups the group names to which the test user will belong + * @return a new User instance + */ + public static User createUserForTesting(Configuration conf, + String name, String[] groups) { + return SecureHadoopUser.createUserForTesting(conf, name, groups); + } + + /** + * Log in the current process using the given configuration keys for the + * credential file and login principal. + * + *

This is only applicable when + * running on secure Hadoop -- see + * org.apache.hadoop.security.SecurityUtil#login(Configuration,String,String,String). + * On regular Hadoop (without security features), this will safely be ignored. + *

+ * + * @param conf The configuration data to use + * @param fileConfKey Property key used to configure path to the credential file + * @param principalConfKey Property key used to configure login principal + * @param localhost Current hostname to use in any credentials + * @throws IOException underlying exception from SecurityUtil.login() call + */ + public static void login(Configuration conf, String fileConfKey, + String principalConfKey, String localhost) throws IOException { + SecureHadoopUser.login(conf, fileConfKey, principalConfKey, localhost); + } + + /** + * Returns whether or not Kerberos authentication is configured for Hadoop. + * For non-secure Hadoop, this always returns false. + * For secure Hadoop, it will return the value from + * {@code UserGroupInformation.isSecurityEnabled()}. + */ + public static boolean isSecurityEnabled() { + return SecureHadoopUser.isSecurityEnabled(HBaseConfiguration.create()); + } + + /** + * Returns whether or not secure authentication is enabled for HBase. Note that + * HBase security requires HDFS security to provide any guarantees, so it is + * recommended that secure HBase should run on secure HDFS. + */ + public static boolean isHBaseSecurityEnabled(Configuration conf) { + return "kerberos".equalsIgnoreCase(conf.get(HBASE_SECURITY_CONF_KEY)); + } + + /* Concrete implementations */ + + /** + * Bridges {@code User} invocations to underlying calls to + * {@link org.apache.hadoop.security.UserGroupInformation} for secure Hadoop + * 0.20 and versions 0.21 and above. + */ + private static class SecureHadoopUser extends User { + private String shortName; + + private SecureHadoopUser() throws IOException { + ugi = UserGroupInformation.getCurrentUser(); + } + + private SecureHadoopUser(UserGroupInformation ugi) { + this.ugi = ugi; + } + + @Override + public String getShortName() { + if (shortName != null) return shortName; + try { + Method m = ugi.getClass().getMethod("getShortUserName"); + shortName = (String) m.invoke(ugi); + } catch (NoSuchMethodException nsme) { + shortName = ugi.getUserName(); + } catch (Exception e) { + throw new RuntimeException("Unexpected error getting user short name", + e); + } + return shortName; + } + + @Override + public T runAs(PrivilegedAction action) { + return ugi.doAs(action); + } + + @Override + public T runAs(PrivilegedExceptionAction action) + throws IOException, InterruptedException { + return ugi.doAs(action); + } + + @Override + public void obtainAuthTokenForJob(Configuration conf, Job job) + throws IOException, InterruptedException { + try { + Class c = Class.forName( + "org.apache.hadoop.hbase.security.token.TokenUtil"); + Methods.call(c, null, "obtainTokenForJob", + new Class[]{Configuration.class, UserGroupInformation.class, + Job.class}, + new Object[]{conf, ugi, job}); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException("Failure loading TokenUtil class, " + +"is secure RPC available?", cnfe); + } catch (IOException ioe) { + throw ioe; + } catch (InterruptedException ie) { + throw ie; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected error calling TokenUtil.obtainAndCacheToken()"); + } + } + + @Override + public void obtainAuthTokenForJob(JobConf job) + throws IOException, InterruptedException { + try { + Class c = Class.forName( + "org.apache.hadoop.hbase.security.token.TokenUtil"); + Methods.call(c, null, "obtainTokenForJob", + new Class[]{JobConf.class, UserGroupInformation.class}, + new Object[]{job, ugi}); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException("Failure loading TokenUtil class, " + +"is secure RPC available?", cnfe); + } catch (IOException ioe) { + throw ioe; + } catch (InterruptedException ie) { + throw ie; + } catch (RuntimeException re) { + throw re; + } catch (Exception e) { + throw new UndeclaredThrowableException(e, + "Unexpected error calling TokenUtil.obtainAndCacheToken()"); + } + } + + /** @see User#createUserForTesting(org.apache.hadoop.conf.Configuration, String, String[]) */ + public static User createUserForTesting(Configuration conf, + String name, String[] groups) { + return new SecureHadoopUser(UserGroupInformation.createUserForTesting(name, groups)); + } + + /** + * Obtain credentials for the current process using the configured + * Kerberos keytab file and principal. + * @see User#login(org.apache.hadoop.conf.Configuration, String, String, String) + * + * @param conf the Configuration to use + * @param fileConfKey Configuration property key used to store the path + * to the keytab file + * @param principalConfKey Configuration property key used to store the + * principal name to login as + * @param localhost the local hostname + */ + public static void login(Configuration conf, String fileConfKey, + String principalConfKey, String localhost) throws IOException { + if (isSecurityEnabled(conf)) { + UserProvider.instantiate(conf).login(fileConfKey, principalConfKey, localhost); + } + } + + /** + * Returns the result of {@code UserGroupInformation.isSecurityEnabled()}. + */ + public static boolean isSecurityEnabled(Configuration conf) { + return !conf.get("hadoop.security.authentication", "simple").equalsIgnoreCase("simple"); + } + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java new file mode 100644 index 0000000..82f686f --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security; + +import java.io.IOException; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.BaseConfigurable; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * Provide an instance of a user. Allows custom {@link User} creation. + */ + +@InterfaceAudience.Private +public class UserProvider extends BaseConfigurable { + + private static final String USER_PROVIDER_CONF_KEY = "hbase.client.userprovider.class"; + + /** + * Instantiate the {@link UserProvider} specified in the configuration and set the passed + * configuration via {@link UserProvider#setConf(Configuration)} + * @param conf to read and set on the created {@link UserProvider} + * @return a {@link UserProvider} ready for use. + */ + public static UserProvider instantiate(Configuration conf) { + Class clazz = + conf.getClass(USER_PROVIDER_CONF_KEY, UserProvider.class, UserProvider.class); + return ReflectionUtils.newInstance(clazz, conf); + } + + /** + * Set the {@link UserProvider} in the given configuration that should be instantiated + * @param conf to update + * @param provider class of the provider to set + */ + public static void setUserProviderForTesting(Configuration conf, + Class provider) { + conf.set(USER_PROVIDER_CONF_KEY, provider.getName()); + } + + /** + * @return the userName for the current logged-in user. + * @throws IOException if the underlying user cannot be obtained + */ + public String getCurrentUserName() throws IOException { + User user = getCurrent(); + return user == null ? null : user.getName(); + } + + /** + * @return true if security is enabled, false otherwise + */ + public boolean isHBaseSecurityEnabled() { + return User.isHBaseSecurityEnabled(this.getConf()); + } + + /** + * @return whether or not Kerberos authentication is configured for Hadoop. For non-secure Hadoop, + * this always returns false. For secure Hadoop, it will return the value + * from {@code UserGroupInformation.isSecurityEnabled()}. + */ + public boolean isHadoopSecurityEnabled() { + return User.isSecurityEnabled(); + } + + /** + * @return the current user within the current execution context + * @throws IOException if the user cannot be loaded + */ + public User getCurrent() throws IOException { + return User.getCurrent(); + } + + /** + * Wraps an underlying {@code UserGroupInformation} instance. + * @param ugi The base Hadoop user + * @return User + */ + public User create(UserGroupInformation ugi) { + return User.create(ugi); + } + + /** + * Log in the current process using the given configuration keys for the credential file and login + * principal. + *

+ * This is only applicable when running on secure Hadoop -- see + * org.apache.hadoop.security.SecurityUtil#login(Configuration,String,String,String). On regular + * Hadoop (without security features), this will safely be ignored. + *

+ * @param fileConfKey Property key used to configure path to the credential file + * @param principalConfKey Property key used to configure login principal + * @param localhost Current hostname to use in any credentials + * @throws IOException underlying exception from SecurityUtil.login() call + */ + public void login(String fileConfKey, String principalConfKey, String localhost) + throws IOException { + User.login(getConf(), fileConfKey, principalConfKey, localhost); + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java index bc6e678..0d8beed 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java @@ -22,31 +22,25 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.TokenSelector; import java.util.Collection; @InterfaceAudience.Private -public class AuthenticationTokenSelector - implements TokenSelector { +public class AuthenticationTokenSelector { private static Log LOG = LogFactory.getLog(AuthenticationTokenSelector.class); public AuthenticationTokenSelector() { } - @Override - public Token selectToken(Text serviceName, - Collection> tokens) { + public HBToken selectToken(Text serviceName, Collection tokens) { if (serviceName != null) { - for (Token ident : tokens) { + for (HBToken ident : tokens) { if (serviceName.equals(ident.getService()) && AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE.equals(ident.getKind())) { if (LOG.isDebugEnabled()) { LOG.debug("Returning token "+ident); } - return (Token)ident; + return ident; } } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/HBToken.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/HBToken.java new file mode 100644 index 0000000..d1edb7c --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/HBToken.java @@ -0,0 +1,153 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.token; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + + +@InterfaceAudience.Private +public class HBToken { + // This will always be a token but older versions of + // Hadoop don't have the notion of a token. + private final Object t; + ReflectionHolder holder; + + private HBToken(byte[] identifier, byte[] password, Text kind, Text service) + throws IllegalAccessException, InvocationTargetException, InstantiationException, + ClassNotFoundException { + this(ReflectionHolder.getHolder().getTokenConstructor().newInstance(identifier, password, kind, service)); + } + + public HBToken(Object t) throws ClassNotFoundException { + this.holder = ReflectionHolder.getHolder(); + this.t = t; + } + + public Text getKind() { + try { + return (Text) holder.getGetKindMethod().invoke(t); + } catch (IllegalAccessException e) { + return null; + } catch (InvocationTargetException e) { + return null; + } + } + + public Text getService() { + try { + return (Text) holder.getGetServiceMethod().invoke(t); + } catch (IllegalAccessException e) { + return null; + } catch (InvocationTargetException e) { + return null; + } + } + + public byte[] getIdentifier() { + try { + return (byte[]) holder.getGetIdentifierMethod().invoke(t); + } catch (IllegalAccessException e) { + return null; + } catch (InvocationTargetException e) { + return null; + } + } + + public byte[] getPassword() { + try { + return (byte[]) holder.getGetPasswordMethod().invoke(t); + } catch (IllegalAccessException e) { + return null; + } catch (InvocationTargetException e) { + return null; + } + } + + public void cancel(Configuration conf) throws InvocationTargetException, IllegalAccessException { + holder.getCancelMethod().invoke(t, conf); + } + + public void addToUGI(UserGroupInformation ugi) { + try { + holder.getUgiAddTokenMethod().invoke(ugi, t); + } catch (IllegalAccessException e) { + } catch (InvocationTargetException e) { + } + } + + public void addToCredentials(Credentials credentials) { + + } + + public Object getToken() { + return t; + } + + ////////////////////////////////////////////////////////////////////////////////////////////// + // + // Static Methods + // + // These are here so that nothing out side of this class has to see the ugliness that is + // java reflection try catch catch catch. + // + ///////////////////////////////////////////////////////////////////////////////////////////// + + public static HBToken createToken(byte[] identifier, byte[] password, Text kind, Text service) { + try { + return new HBToken(identifier, password, kind, service); + } catch (InvocationTargetException e) { + return null; + } catch (ClassNotFoundException e) { + return null; + } catch (InstantiationException e) { + return null; + } catch (IllegalAccessException e) { + return null; + } + } + + public static Collection getTokensFromUGI(UserGroupInformation ugi) { + try { + Collection c = (Collection) ReflectionHolder.getHolder().getUgiGetTokensMethod().invoke(ugi); + List hbTokens = new ArrayList<>(c.size()); + for (Object t:c) { + hbTokens.add(new HBToken(t)); + } + return hbTokens; + } catch (NullPointerException e) { + return Collections.EMPTY_LIST; + } catch (InvocationTargetException e) { + return Collections.EMPTY_LIST; + } catch (IllegalAccessException e) { + return Collections.EMPTY_LIST; + } catch (ClassNotFoundException e) { + return Collections.EMPTY_LIST; + } + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/ReflectionHolder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/ReflectionHolder.java new file mode 100644 index 0000000..c6b73c1 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/ReflectionHolder.java @@ -0,0 +1,106 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.token; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; + +public class ReflectionHolder { + private final Class tokenClass; + private final Constructor tokenFourCon; + + private final Method getKindMethod; + private final Method getServiceMethod; + private final Method getIdentifierMethod; + private final Method getPasswordMethod; + private final Method cancelMethod; + + private final Class ugiClass = UserGroupInformation.class; + private final Method ugiAddTokenMethod; + private final Method ugiGetTokensMethod; + + ReflectionHolder() { + Class tokenClassTemp; + Constructor tokenFourConTemp; + + try { + tokenClassTemp = Class.forName("org.apache.hadoop.security.token.Token"); + } catch (ClassNotFoundException e) { + tokenClassTemp = null; + } + this.tokenClass = tokenClassTemp; + + try { + tokenFourConTemp = tokenClass.getConstructor(new Class[]{ byte[].class, byte[].class, Text.class, Text.class}); + } catch (NoSuchMethodException e) { + tokenFourConTemp = null; + } catch (NullPointerException npe) { + tokenFourConTemp = null; + } + + this.tokenFourCon = tokenFourConTemp; + + getServiceMethod = getMethod(tokenClass, "getService"); + getIdentifierMethod = getMethod(tokenClass, "getIdentifier"); + getKindMethod = getMethod(tokenClass, "getKind"); + getPasswordMethod = getMethod(tokenClass, "getPassword"); + cancelMethod = getMethod(tokenClass, "cancel", Configuration.class); + + ugiAddTokenMethod = getMethod(ugiClass, "addToken", tokenClass); + ugiGetTokensMethod = getMethod(ugiClass, "getTokens"); + } + + private Method getMethod(Class klass, String methodName, Class... params) { + Method methodTemp; + try { + methodTemp = klass.getMethod(methodName, params); + } catch (NoSuchMethodException e) { + methodTemp = null; + } catch (NullPointerException npe){ + methodTemp = null; + } + return methodTemp; + } + + public Class getTokenClass() { + return tokenClass; + } + + public Method getGetKindMethod() { return getKindMethod; } + public Method getGetServiceMethod() { return getServiceMethod; } + public Method getGetIdentifierMethod() { return getIdentifierMethod; } + public Method getGetPasswordMethod() { return getPasswordMethod; } + public Method getCancelMethod() { return cancelMethod; } + public Constructor getTokenConstructor() { return tokenFourCon; } + public Method getUgiAddTokenMethod() { return ugiAddTokenMethod; } + public Method getUgiGetTokensMethod() { return ugiGetTokensMethod; } + + private static enum Singleton { + INSTANCE; + ReflectionHolder holder = new ReflectionHolder(); + } + + public static ReflectionHolder getHolder() { + return Singleton.INSTANCE.holder; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java index 8a349db..9d758a0 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java @@ -37,7 +37,6 @@ import org.apache.hadoop.io.compress.CompressionOutputStream; import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Decompressor; import org.apache.hadoop.io.compress.DefaultCodec; -import org.apache.hadoop.io.compress.DoNotPool; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.util.ReflectionUtils; @@ -49,6 +48,16 @@ import org.apache.hadoop.util.ReflectionUtils; public final class Compression { static final Log LOG = LogFactory.getLog(Compression.class); + private static Class doNotPoolClass; + + static { + try { + doNotPoolClass = Class.forName("org.apache.hadoop.io.compress.DoNotPool"); + } catch (ClassNotFoundException e) { + doNotPoolClass = null; + } + } + /** * Prevent the instantiation of class. */ @@ -343,7 +352,7 @@ public final class Compression { if (decompressor != null) { if (LOG.isTraceEnabled()) LOG.trace("Returning decompressor " + decompressor + " to pool."); CodecPool.returnDecompressor(decompressor); - if (decompressor.getClass().isAnnotationPresent(DoNotPool.class)) { + if (doNotPoolClass == null || decompressor.getClass().isAnnotationPresent(doNotPoolClass)) { if (LOG.isTraceEnabled()) LOG.trace("Ending decompressor " + decompressor); decompressor.end(); } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/security/User.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/security/User.java deleted file mode 100644 index 901c57c..0000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/security/User.java +++ /dev/null @@ -1,367 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.security; - -import java.io.IOException; -import java.lang.reflect.UndeclaredThrowableException; -import java.security.PrivilegedAction; -import java.security.PrivilegedExceptionAction; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.util.Methods; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; - -/** - * Wrapper to abstract out usage of user and group information in HBase. - * - *

- * This class provides a common interface for interacting with user and group - * information across changing APIs in different versions of Hadoop. It only - * provides access to the common set of functionality in - * {@link org.apache.hadoop.security.UserGroupInformation} currently needed by - * HBase, but can be extended as needs change. - *

- */ -@InterfaceAudience.Private -public abstract class User { - public static final String HBASE_SECURITY_CONF_KEY = - "hbase.security.authentication"; - - protected UserGroupInformation ugi; - - public UserGroupInformation getUGI() { - return ugi; - } - - /** - * Returns the full user name. For Kerberos principals this will include - * the host and realm portions of the principal name. - * @return User full name. - */ - public String getName() { - return ugi.getUserName(); - } - - /** - * Returns the list of groups of which this user is a member. On secure - * Hadoop this returns the group information for the user as resolved on the - * server. For 0.20 based Hadoop, the group names are passed from the client. - */ - public String[] getGroupNames() { - return ugi.getGroupNames(); - } - - /** - * Returns the shortened version of the user name -- the portion that maps - * to an operating system user name. - * @return Short name - */ - public abstract String getShortName(); - - /** - * Executes the given action within the context of this user. - */ - public abstract T runAs(PrivilegedAction action); - - /** - * Executes the given action within the context of this user. - */ - public abstract T runAs(PrivilegedExceptionAction action) - throws IOException, InterruptedException; - - /** - * Requests an authentication token for this user and stores it in the - * user's credentials. - * - * @throws IOException - */ - public abstract void obtainAuthTokenForJob(Configuration conf, Job job) - throws IOException, InterruptedException; - - /** - * Requests an authentication token for this user and stores it in the - * user's credentials. - * - * @throws IOException - */ - public abstract void obtainAuthTokenForJob(JobConf job) - throws IOException, InterruptedException; - - /** - * Returns the Token of the specified kind associated with this user, - * or null if the Token is not present. - * - * @param kind the kind of token - * @param service service on which the token is supposed to be used - * @return the token of the specified kind. - */ - public Token getToken(String kind, String service) throws IOException { - for (Token token: ugi.getTokens()) { - if (token.getKind().toString().equals(kind) && - (service != null && token.getService().toString().equals(service))) - { - return token; - } - } - return null; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - return ugi.equals(((User) o).ugi); - } - - @Override - public int hashCode() { - return ugi.hashCode(); - } - - @Override - public String toString() { - return ugi.toString(); - } - - /** - * Returns the {@code User} instance within current execution context. - */ - public static User getCurrent() throws IOException { - User user = new SecureHadoopUser(); - if (user.getUGI() == null) { - return null; - } - return user; - } - - /** - * Executes the given action as the login user - * @param action - * @return the result of the action - * @throws IOException - * @throws InterruptedException - */ - @SuppressWarnings({ "rawtypes", "unchecked" }) - public static T runAsLoginUser(PrivilegedExceptionAction action) throws IOException { - try { - Class c = Class.forName("org.apache.hadoop.security.SecurityUtil"); - Class [] types = new Class[]{PrivilegedExceptionAction.class}; - Object[] args = new Object[]{action}; - return (T) Methods.call(c, null, "doAsLoginUser", types, args); - } catch (Throwable e) { - throw new IOException(e); - } - } - - /** - * Wraps an underlying {@code UserGroupInformation} instance. - * @param ugi The base Hadoop user - * @return User - */ - public static User create(UserGroupInformation ugi) { - if (ugi == null) { - return null; - } - return new SecureHadoopUser(ugi); - } - - /** - * Generates a new {@code User} instance specifically for use in test code. - * @param name the full username - * @param groups the group names to which the test user will belong - * @return a new User instance - */ - public static User createUserForTesting(Configuration conf, - String name, String[] groups) { - return SecureHadoopUser.createUserForTesting(conf, name, groups); - } - - /** - * Log in the current process using the given configuration keys for the - * credential file and login principal. - * - *

This is only applicable when - * running on secure Hadoop -- see - * org.apache.hadoop.security.SecurityUtil#login(Configuration,String,String,String). - * On regular Hadoop (without security features), this will safely be ignored. - *

- * - * @param conf The configuration data to use - * @param fileConfKey Property key used to configure path to the credential file - * @param principalConfKey Property key used to configure login principal - * @param localhost Current hostname to use in any credentials - * @throws IOException underlying exception from SecurityUtil.login() call - */ - public static void login(Configuration conf, String fileConfKey, - String principalConfKey, String localhost) throws IOException { - SecureHadoopUser.login(conf, fileConfKey, principalConfKey, localhost); - } - - /** - * Returns whether or not Kerberos authentication is configured for Hadoop. - * For non-secure Hadoop, this always returns false. - * For secure Hadoop, it will return the value from - * {@code UserGroupInformation.isSecurityEnabled()}. - */ - public static boolean isSecurityEnabled() { - return SecureHadoopUser.isSecurityEnabled(); - } - - /** - * Returns whether or not secure authentication is enabled for HBase. Note that - * HBase security requires HDFS security to provide any guarantees, so it is - * recommended that secure HBase should run on secure HDFS. - */ - public static boolean isHBaseSecurityEnabled(Configuration conf) { - return "kerberos".equalsIgnoreCase(conf.get(HBASE_SECURITY_CONF_KEY)); - } - - /* Concrete implementations */ - - /** - * Bridges {@code User} invocations to underlying calls to - * {@link org.apache.hadoop.security.UserGroupInformation} for secure Hadoop - * 0.20 and versions 0.21 and above. - */ - private static class SecureHadoopUser extends User { - private String shortName; - - private SecureHadoopUser() throws IOException { - ugi = UserGroupInformation.getCurrentUser(); - } - - private SecureHadoopUser(UserGroupInformation ugi) { - this.ugi = ugi; - } - - @Override - public String getShortName() { - if (shortName != null) return shortName; - try { - shortName = ugi.getShortUserName(); - return shortName; - } catch (Exception e) { - throw new RuntimeException("Unexpected error getting user short name", - e); - } - } - - @Override - public T runAs(PrivilegedAction action) { - return ugi.doAs(action); - } - - @Override - public T runAs(PrivilegedExceptionAction action) - throws IOException, InterruptedException { - return ugi.doAs(action); - } - - @Override - public void obtainAuthTokenForJob(Configuration conf, Job job) - throws IOException, InterruptedException { - try { - Class c = Class.forName( - "org.apache.hadoop.hbase.security.token.TokenUtil"); - Methods.call(c, null, "obtainTokenForJob", - new Class[]{Configuration.class, UserGroupInformation.class, - Job.class}, - new Object[]{conf, ugi, job}); - } catch (ClassNotFoundException cnfe) { - throw new RuntimeException("Failure loading TokenUtil class, " - +"is secure RPC available?", cnfe); - } catch (IOException ioe) { - throw ioe; - } catch (InterruptedException ie) { - throw ie; - } catch (RuntimeException re) { - throw re; - } catch (Exception e) { - throw new UndeclaredThrowableException(e, - "Unexpected error calling TokenUtil.obtainAndCacheToken()"); - } - } - - @Override - public void obtainAuthTokenForJob(JobConf job) - throws IOException, InterruptedException { - try { - Class c = Class.forName( - "org.apache.hadoop.hbase.security.token.TokenUtil"); - Methods.call(c, null, "obtainTokenForJob", - new Class[]{JobConf.class, UserGroupInformation.class}, - new Object[]{job, ugi}); - } catch (ClassNotFoundException cnfe) { - throw new RuntimeException("Failure loading TokenUtil class, " - +"is secure RPC available?", cnfe); - } catch (IOException ioe) { - throw ioe; - } catch (InterruptedException ie) { - throw ie; - } catch (RuntimeException re) { - throw re; - } catch (Exception e) { - throw new UndeclaredThrowableException(e, - "Unexpected error calling TokenUtil.obtainAndCacheToken()"); - } - } - - /** @see User#createUserForTesting(org.apache.hadoop.conf.Configuration, String, String[]) */ - public static User createUserForTesting(Configuration conf, - String name, String[] groups) { - return new SecureHadoopUser(UserGroupInformation.createUserForTesting(name, groups)); - } - - /** - * Obtain credentials for the current process using the configured - * Kerberos keytab file and principal. - * @see User#login(org.apache.hadoop.conf.Configuration, String, String, String) - * - * @param conf the Configuration to use - * @param fileConfKey Configuration property key used to store the path - * to the keytab file - * @param principalConfKey Configuration property key used to store the - * principal name to login as - * @param localhost the local hostname - */ - public static void login(Configuration conf, String fileConfKey, - String principalConfKey, String localhost) throws IOException { - if (isSecurityEnabled()) { - SecurityUtil.login(conf, fileConfKey, principalConfKey, localhost); - } - } - - /** - * Returns the result of {@code UserGroupInformation.isSecurityEnabled()}. - */ - public static boolean isSecurityEnabled() { - return UserGroupInformation.isSecurityEnabled(); - } - } -} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java deleted file mode 100644 index 82f686f..0000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.security; - -import java.io.IOException; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.BaseConfigurable; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ReflectionUtils; - -/** - * Provide an instance of a user. Allows custom {@link User} creation. - */ - -@InterfaceAudience.Private -public class UserProvider extends BaseConfigurable { - - private static final String USER_PROVIDER_CONF_KEY = "hbase.client.userprovider.class"; - - /** - * Instantiate the {@link UserProvider} specified in the configuration and set the passed - * configuration via {@link UserProvider#setConf(Configuration)} - * @param conf to read and set on the created {@link UserProvider} - * @return a {@link UserProvider} ready for use. - */ - public static UserProvider instantiate(Configuration conf) { - Class clazz = - conf.getClass(USER_PROVIDER_CONF_KEY, UserProvider.class, UserProvider.class); - return ReflectionUtils.newInstance(clazz, conf); - } - - /** - * Set the {@link UserProvider} in the given configuration that should be instantiated - * @param conf to update - * @param provider class of the provider to set - */ - public static void setUserProviderForTesting(Configuration conf, - Class provider) { - conf.set(USER_PROVIDER_CONF_KEY, provider.getName()); - } - - /** - * @return the userName for the current logged-in user. - * @throws IOException if the underlying user cannot be obtained - */ - public String getCurrentUserName() throws IOException { - User user = getCurrent(); - return user == null ? null : user.getName(); - } - - /** - * @return true if security is enabled, false otherwise - */ - public boolean isHBaseSecurityEnabled() { - return User.isHBaseSecurityEnabled(this.getConf()); - } - - /** - * @return whether or not Kerberos authentication is configured for Hadoop. For non-secure Hadoop, - * this always returns false. For secure Hadoop, it will return the value - * from {@code UserGroupInformation.isSecurityEnabled()}. - */ - public boolean isHadoopSecurityEnabled() { - return User.isSecurityEnabled(); - } - - /** - * @return the current user within the current execution context - * @throws IOException if the user cannot be loaded - */ - public User getCurrent() throws IOException { - return User.getCurrent(); - } - - /** - * Wraps an underlying {@code UserGroupInformation} instance. - * @param ugi The base Hadoop user - * @return User - */ - public User create(UserGroupInformation ugi) { - return User.create(ugi); - } - - /** - * Log in the current process using the given configuration keys for the credential file and login - * principal. - *

- * This is only applicable when running on secure Hadoop -- see - * org.apache.hadoop.security.SecurityUtil#login(Configuration,String,String,String). On regular - * Hadoop (without security features), this will safely be ignored. - *

- * @param fileConfKey Property key used to configure path to the credential file - * @param principalConfKey Property key used to configure login principal - * @param localhost Current hostname to use in any credentials - * @throws IOException underlying exception from SecurityUtil.login() call - */ - public void login(String fileConfKey, String principalConfKey, String localhost) - throws IOException { - User.login(getConf(), fileConfKey, principalConfKey, localhost); - } -} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java index 7b837af..2754e55 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java @@ -155,7 +155,8 @@ public class CoprocessorClassLoader extends ClassLoaderBase { FileSystem fs = FileSystem.getLocal(conf); fs.delete(parentDir, true); // it's ok if the dir doesn't exist now parentDirLockSet.add(parentDirStr); - if (!fs.mkdirs(parentDir) && !fs.getFileStatus(parentDir).isDirectory()) { + // isDir is deprectated so we might have to use reflection here in the future. + if (!fs.mkdirs(parentDir) && !fs.getFileStatus(parentDir).isDir()) { throw new RuntimeException("Failed to create local dir " + parentDirStr + ", CoprocessorClassLoader failed to init"); } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java index e434558..3f61d97 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java @@ -185,7 +185,7 @@ public class DynamicClassLoader extends ClassLoaderBase { } for (FileStatus status: statuses) { - if (status.isDirectory()) continue; // No recursive lookup + if (status.isDir()) continue; // No recursive lookup Path path = status.getPath(); String fileName = path.getName(); if (!fileName.endsWith(".jar")) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java index f662e42..6d0ce30 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.token.HBToken; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.io.Text; @@ -238,11 +239,11 @@ public class TableMapReduceUtil { try { // login the server principal (if using secure Hadoop) User user = userProvider.getCurrent(); - Token authToken = getAuthToken(job, user); + HBToken authToken = getAuthToken(job, user); if (authToken == null) { user.obtainAuthTokenForJob(job); } else { - job.getCredentials().addToken(authToken.getService(), authToken); + authToken.addToCredentials(job.getCredentials()); } } catch (InterruptedException ie) { ie.printStackTrace(); @@ -255,12 +256,12 @@ public class TableMapReduceUtil { * Get the authentication token of the user for the cluster specified in the configuration * @return null if the user does not have the token, otherwise the auth token for the cluster. */ - private static Token getAuthToken(Configuration conf, User user) + private static HBToken getAuthToken(Configuration conf, User user) throws IOException, InterruptedException { ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "mr-init-credentials", null); try { String clusterId = ZKClusterId.readClusterIdZNode(zkw); - return new AuthenticationTokenSelector().selectToken(new Text(clusterId), user.getUGI().getTokens()); + return new AuthenticationTokenSelector().selectToken(new Text(clusterId), user.getTokens()); } catch (KeeperException e) { throw new IOException(e); } finally { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index 568384c..e424fdf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -55,8 +55,12 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector; +import org.apache.hadoop.hbase.security.token.HBToken; import org.apache.hadoop.hbase.util.Base64; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Classes; +import org.apache.hadoop.hbase.util.ExceptionUtil; +import org.apache.hadoop.hbase.util.Methods; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; @@ -65,6 +69,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.StringUtils; import org.apache.zookeeper.KeeperException; @@ -465,11 +470,22 @@ public class TableMapReduceUtil { private static void obtainAuthTokenForJob(Job job, Configuration conf, User user) throws IOException, InterruptedException { - Token authToken = getAuthToken(conf, user); + HBToken authToken = getAuthToken(conf, user); if (authToken == null) { user.obtainAuthTokenForJob(conf, job); } else { - job.getCredentials().addToken(authToken.getService(), authToken); + + try { + Credentials creds = job.getCredentials(); + Class credentialsKlass = creds.getClass(); + Class tokenKlass = Class.forName("org.apache.hadoop.security.token.Token"); + Methods.call(credentialsKlass, creds, "addToken", new Class[] { Text.class, tokenKlass }, + new Object[] { authToken.getService(), authToken.getToken() }); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException("Unknown Exception obtaining token", e); + } } } @@ -477,12 +493,12 @@ public class TableMapReduceUtil { * Get the authentication token of the user for the cluster specified in the configuration * @return null if the user does not have the token, otherwise the auth token for the cluster. */ - private static Token getAuthToken(Configuration conf, User user) + private static HBToken getAuthToken(Configuration conf, User user) throws IOException, InterruptedException { ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "mr-init-credentials", null); try { String clusterId = ZKClusterId.readClusterIdZNode(zkw); - return new AuthenticationTokenSelector().selectToken(new Text(clusterId), user.getUGI().getTokens()); + return new AuthenticationTokenSelector().selectToken(new Text(clusterId), user.getTokens()); } catch (KeeperException e) { throw new IOException(e); } finally { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java index b7b8e7b..cfa6936 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.security.SecureBulkLoadUtil; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.token.FsDelegationToken; +import org.apache.hadoop.hbase.security.token.HBToken; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSHDFSUtils; import org.apache.hadoop.hbase.util.Methods; @@ -191,16 +192,16 @@ public class SecureBulkLoadEndpoint extends SecureBulkLoadService for(ClientProtos.BulkLoadHFileRequest.FamilyPath el : request.getFamilyPathList()) { familyPaths.add(new Pair(el.getFamily().toByteArray(),el.getPath())); } - final Token userToken = - new Token(request.getFsToken().getIdentifier().toByteArray(), - request.getFsToken().getPassword().toByteArray(), - new Text(request.getFsToken().getKind()), - new Text(request.getFsToken().getService())); + final HBToken userToken = HBToken.createToken(request.getFsToken().getIdentifier().toByteArray(), + request.getFsToken().getPassword().toByteArray(), + new Text(request.getFsToken().getKind()), + new Text(request.getFsToken().getService())); + final String bulkToken = request.getBulkToken(); User user = getActiveUser(); final UserGroupInformation ugi = user.getUGI(); if(userToken != null) { - ugi.addToken(userToken); + userToken.addToUGI(ugi); } else if (userProvider.isHadoopSecurityEnabled()) { //we allow this to pass through in "simple" security mode //for mini cluster testing @@ -235,10 +236,10 @@ public class SecureBulkLoadEndpoint extends SecureBulkLoadService done.run(null); return; } - Token targetFsToken = targetfsDelegationToken.getUserToken(); + HBToken targetFsToken = targetfsDelegationToken.getUserToken(); if (targetFsToken != null && (userToken == null || !targetFsToken.getService().equals(userToken.getService()))) { - ugi.addToken(targetFsToken); + targetFsToken.addToUGI(ugi); } loaded = ugi.doAs(new PrivilegedAction() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/FsDelegationToken.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/FsDelegationToken.java index 5faf91f..5232200 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/FsDelegationToken.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/FsDelegationToken.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.util.Methods; import org.apache.hadoop.security.token.Token; /** @@ -42,7 +43,7 @@ public class FsDelegationToken { private final String renewer; private boolean hasForwardedToken = false; - private Token userToken = null; + private HBToken userToken = null; private FileSystem fs = null; /* @@ -69,8 +70,10 @@ public class FsDelegationToken { if (userToken == null) { hasForwardedToken = false; try { - userToken = fs.getDelegationToken(renewer); - } catch (NullPointerException npe) { + + Object t = Methods.call(FileSystem.class, fs, "getDelegationToken", new Class[]{String.class}, new Object[] {renewer}); + userToken = new HBToken(t); + } catch (Exception npe) { // we need to handle NullPointerException in case HADOOP-10009 is missing LOG.error("Failed to get token for " + renewer); } @@ -112,7 +115,7 @@ public class FsDelegationToken { /** * @return the delegation token acquired, or null in case it was not acquired */ - public Token getUserToken() { + public HBToken getUserToken() { return userToken; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestHBaseSaslRpcClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestHBaseSaslRpcClient.java index 21450a2..d236114 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestHBaseSaslRpcClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestHBaseSaslRpcClient.java @@ -43,6 +43,7 @@ import javax.security.sasl.RealmCallback; import javax.security.sasl.RealmChoiceCallback; import javax.security.sasl.SaslClient; +import org.apache.hadoop.hbase.security.token.HBToken; import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.security.HBaseSaslRpcClient.SaslClientCallbackHandler; @@ -79,7 +80,7 @@ public class TestHBaseSaslRpcClient { @Test public void testSaslQOPNotEmpty() throws Exception { - Token token = createTokenMockWithCredentials(DEFAULT_USER_NAME, + HBToken token = createTokenMockWithCredentials(DEFAULT_USER_NAME, DEFAULT_USER_PASSWORD); // default QOP is authentication new HBaseSaslRpcClient(AuthMethod.DIGEST, token, "principal/host@DOMAIN.COM", false); @@ -105,7 +106,7 @@ public class TestHBaseSaslRpcClient { @Test public void testSaslClientCallbackHandler() throws UnsupportedCallbackException { - final Token token = createTokenMock(); + final HBToken token = createTokenMock(); when(token.getIdentifier()).thenReturn(DEFAULT_USER_NAME.getBytes()); when(token.getPassword()).thenReturn(DEFAULT_USER_PASSWORD.getBytes()); @@ -125,7 +126,7 @@ public class TestHBaseSaslRpcClient { @Test public void testSaslClientCallbackHandlerWithException() { - final Token token = createTokenMock(); + final HBToken token = createTokenMock(); when(token.getIdentifier()).thenReturn(DEFAULT_USER_NAME.getBytes()); when(token.getPassword()).thenReturn(DEFAULT_USER_PASSWORD.getBytes()); final SaslClientCallbackHandler saslClCallbackHandler = new SaslClientCallbackHandler(token); @@ -292,10 +293,10 @@ public class TestHBaseSaslRpcClient { return new HBaseSaslRpcClient(AuthMethod.KERBEROS, createTokenMock(), principal, false); } - private Token createTokenMockWithCredentials( + private HBToken createTokenMockWithCredentials( String principal, String password) throws IOException { - Token token = createTokenMock(); + HBToken token = createTokenMock(); if (!Strings.isNullOrEmpty(principal) && !Strings.isNullOrEmpty(password)) { when(token.getIdentifier()).thenReturn(DEFAULT_USER_NAME.getBytes()); when(token.getPassword()).thenReturn(DEFAULT_USER_PASSWORD.getBytes()); @@ -309,7 +310,12 @@ public class TestHBaseSaslRpcClient { } @SuppressWarnings("unchecked") - private Token createTokenMock() { - return mock(Token.class); + private HBToken createTokenMock() { + try { + return new HBToken(mock(Token.class)); + } catch (ClassNotFoundException e) { + assertTrue("Couldn't find the token class", false); + return null; + } } } diff --git a/pom.xml b/pom.xml index bf5dafd..a019b72 100644 --- a/pom.xml +++ b/pom.xml @@ -1479,8 +1479,7 @@ - !hadoop.profile - + !hadoop.profile hbase-hadoop2-compat @@ -1628,6 +1627,7 @@ +