diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 085bed9..4edf200 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2705,6 +2705,13 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_COMPACTOR_COMPACT_MM("hive.compactor.compact.insert.only", true, "Whether the compactor should compact insert-only tables. A safety switch."), + COMPACTOR_CRUD_QUERY_BASED("hive.compactor.crud.query.based", false, + "Means Major compaction on full CRUD tables is done as a query, " + + "and minor compaction will be disabled."), + SPLIT_GROUPING_MODE("hive.split.grouping.mode", "query", new StringSet("query", "compactor"), + "This is set to compactor from within the query based compactor. This enables the Tez SplitGrouper " + + "to group splits based on their bucket number, so that all rows from different bucket files " + + " for the same bucket number can end up in the same bucket file after the compaction."), /** * @deprecated Use MetastoreConf.COMPACTOR_HISTORY_RETENTION_SUCCEEDED */ @@ -2754,6 +2761,11 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal MERGE_CARDINALITY_VIOLATION_CHECK("hive.merge.cardinality.check", true, "Set to true to ensure that each SQL Merge statement ensures that for each row in the target\n" + "table there is at most 1 matching row in the source table per SQL Specification."), + MERGE_SPLIT_UPDATE("hive.merge.split.update", false, + "If true, SQL Merge statement will handle WHEN MATCHED UPDATE by splitting it into 2\n" + + "branches of a multi-insert, representing delete of existing row and an insert of\n" + + "the new version of the row. Updating bucketing and partitioning columns should\n" + + "only be permitted if this is true."), OPTIMIZE_ACID_META_COLUMNS("hive.optimize.acid.meta.columns", true, "If true, don't decode Acid metadata columns from storage unless" + " they are needed."), @@ -3487,6 +3499,16 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "For example: (&(objectClass=group)(objectClass=top)(instanceType=4)(cn=Domain*)) \n" + "(&(objectClass=person)(|(sAMAccountName=admin)(|(memberOf=CN=Domain Admins,CN=Users,DC=domain,DC=com)" + "(memberOf=CN=Administrators,CN=Builtin,DC=domain,DC=com))))"), + HIVE_SERVER2_PLAIN_LDAP_BIND_USER("hive.server2.authentication.ldap.binddn", null, + "The user with which to bind to the LDAP server, and search for the full domain name " + + "of the user being authenticated.\n" + + "This should be the full domain name of the user, and should have search access across all " + + "users in the LDAP tree.\n" + + "If not specified, then the user being authenticated will be used as the bind user.\n" + + "For example: CN=bindUser,CN=Users,DC=subdomain,DC=domain,DC=com"), + HIVE_SERVER2_PLAIN_LDAP_BIND_PASSWORD("hive.server2.authentication.ldap.bindpw", null, + "The password for the bind user, to be used to search for the full name of the user being authenticated.\n" + + "If the username is specified, this parameter must also be specified."), HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS("hive.server2.custom.authentication.class", null, "Custom authentication class. Used when property\n" + "'hive.server2.authentication' is set to 'CUSTOM'. Provided class\n" + @@ -4443,7 +4465,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "How frequently to check for idle Spark sessions. Minimum value is 60 seconds."), NWAYJOINREORDER("hive.reorder.nway.joins", true, "Runs reordering of tables within single n-way join (i.e.: picks streamtable)"), - HIVE_MERGE_NWAY_JOINS("hive.merge.nway.joins", true, + HIVE_MERGE_NWAY_JOINS("hive.merge.nway.joins", false, "Merge adjacent joins into a single n-way join"), HIVE_LOG_N_RECORDS("hive.log.every.n.records", 0L, new RangeValidator(0L, null), "If value is greater than 0 logs in fixed intervals of size n rather than exponentially."), diff --git service/pom.xml service/pom.xml index eca6f3b..3c36070 100644 --- service/pom.xml +++ service/pom.xml @@ -323,6 +323,11 @@ ${basedir}/src/java ${basedir}/src/test + + + ${basedir}/src/test/resources + + ${project.build.directory} diff --git service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java index 73bbb6b..b3c362f 100644 --- service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java +++ service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java @@ -18,9 +18,10 @@ package org.apache.hive.service.auth; import javax.security.sasl.AuthenticationException; - +import javax.naming.NamingException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import java.io.IOException; import java.util.Iterator; import java.util.List; import org.apache.commons.lang.StringUtils; @@ -68,9 +69,36 @@ public LdapAuthenticationProviderImpl(HiveConf conf) { @Override public void Authenticate(String user, String password) throws AuthenticationException { DirSearch search = null; + String bindUser = this.conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_USER); + String bindPassword = null; + try { + char[] rawPassword = this.conf.getPassword(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_PASSWORD.toString()); + if (rawPassword != null) { + bindPassword = new String(rawPassword); + } + } catch (IOException e) { + bindPassword = null; + } + boolean usedBind = bindUser != null && bindPassword != null; + if (!usedBind) { + // If no bind user or bind password was specified, + // we assume the user we are authenticating has the ability to search + // the LDAP tree, so we use it as the "binding" account. + // This is the way it worked before bind users were allowed in the LDAP authenticator, + // so we keep existing systems working. + bindUser = user; + bindPassword = password; + } try { - search = createDirSearch(user, password); + search = createDirSearch(bindUser, bindPassword); applyFilter(search, user); + if (usedBind) { + // If we used the bind user, then we need to authenticate again, + // this time using the full user name we got during the bind process. + createDirSearch(search.findUserDn(user), password); + } + } catch (NamingException e) { + throw new AuthenticationException("Unable to find the user in the LDAP tree. " + e.getMessage()); } finally { ServiceUtils.cleanup(LOG, search); } diff --git service/src/test/org/apache/hive/service/auth/TestLdapAuthenticationProviderImpl.java service/src/test/org/apache/hive/service/auth/TestLdapAuthenticationProviderImpl.java index 43453fa..7a1879f 100644 --- service/src/test/org/apache/hive/service/auth/TestLdapAuthenticationProviderImpl.java +++ service/src/test/org/apache/hive/service/auth/TestLdapAuthenticationProviderImpl.java @@ -22,6 +22,7 @@ import javax.naming.NamingException; import javax.security.sasl.AuthenticationException; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hive.service.auth.ldap.DirSearch; import org.apache.hive.service.auth.ldap.DirSearchFactory; import org.apache.hive.service.auth.ldap.LdapSearchFactory; @@ -324,6 +325,118 @@ public void testAuthenticateWhenUserMembershipKeyFilter2x2PatternsPasses() throw verify(search, atLeastOnce()).close(); } + @Test + public void testAuthenticateWithBindInCredentialFilePasses() throws AuthenticationException, NamingException { + String bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String bindPass = "testPassword"; + String authFullUser = "cn=user1,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String authUser = "user1"; + String authPass = "Blah"; + String tmpDir = System.getProperty("build.dir"); + String credentialsPath = "jceks://file" + tmpDir + "/test-classes/creds/test.jceks" + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_USER, bindUser); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, credentialsPath); + + System.out.println(tmpDir); + + when(search.findUserDn(eq(authUser))).thenReturn(authFullUser); + + auth = new LdapAuthenticationProviderImpl(conf, factory); + auth.Authenticate(authUser, authPass); + + verify(factory, times(1)).getInstance(isA(HiveConf.class), eq(bindUser), eq(bindPass)); + verify(factory, times(1)).getInstance(isA(HiveConf.class), eq(authFullUser), eq(authPass)); + verify(search, times(1)).findUserDn(eq(authUser)); + } + + @Test + public void testAuthenticateWithBindInMissingCredentialFilePasses() throws AuthenticationException, NamingException { + String bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String authUser = "user1"; + String authPass = "Blah"; + String tmpDir = System.getProperty("build.dir"); + String credentialsPath = "jceks://file" + tmpDir + "/test-classes/creds/nonExistent.jceks" + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_USER, bindUser); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, credentialsPath); + + auth = new LdapAuthenticationProviderImpl(conf, factory); + auth.Authenticate(authUser, authPass); + + verify(factory, times(1)).getInstance(isA(HiveConf.class), eq(authUser), eq(authPass)); + } + + @Test + public void testAuthenticateWithBindUserPasses() throws AuthenticationException, NamingException { + String bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String bindPass = "Blah"; + String authFullUser = "cn=user1,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String authUser = "user1"; + String authPass = "Blah2"; + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_USER, bindUser); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_PASSWORD, bindPass); + + when(search.findUserDn(eq(authUser))).thenReturn(authFullUser); + + auth = new LdapAuthenticationProviderImpl(conf, factory); + auth.Authenticate(authUser, authPass); + + verify(factory, times(1)).getInstance(isA(HiveConf.class), eq(bindUser), eq(bindPass)); + verify(factory, times(1)).getInstance(isA(HiveConf.class), eq(authFullUser), eq(authPass)); + verify(search, times(1)).findUserDn(eq(authUser)); + } + + @Test + public void testAuthenticateWithBindUserFailsOnAuthentication() throws AuthenticationException, NamingException { + String bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String bindPass = "Blah"; + String authFullUser = "cn=user1,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String authUser = "user1"; + String authPass = "Blah2"; + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_USER, bindUser); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_PASSWORD, bindPass); + + thrown.expect(AuthenticationException.class); + when(factory.getInstance(any(HiveConf.class), eq(authFullUser), eq(authPass))). + thenThrow(AuthenticationException.class); + when(search.findUserDn(eq(authUser))).thenReturn(authFullUser); + + auth = new LdapAuthenticationProviderImpl(conf, factory); + auth.Authenticate(authUser, authPass); + } + + @Test + public void testAuthenticateWithBindUserFailsOnGettingDn() throws AuthenticationException, NamingException { + String bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String bindPass = "Blah"; + String authFullUser = "cn=user1,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String authUser = "user1"; + String authPass = "Blah2"; + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_USER, bindUser); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_PASSWORD, bindPass); + + thrown.expect(AuthenticationException.class); + when(search.findUserDn(eq(authUser))).thenThrow(NamingException.class); + + auth = new LdapAuthenticationProviderImpl(conf, factory); + auth.Authenticate(authUser, authPass); + } + + @Test + public void testAuthenticateWithBindUserFailsOnBinding() throws AuthenticationException { + String bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com"; + String bindPass = "Blah"; + String authUser = "user1"; + String authPass = "Blah2"; + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_USER, bindUser); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BIND_PASSWORD, bindPass); + + thrown.expect(AuthenticationException.class); + when(factory.getInstance(any(HiveConf.class), eq(bindUser), eq(bindPass))).thenThrow(AuthenticationException.class); + + auth = new LdapAuthenticationProviderImpl(conf, factory); + auth.Authenticate(authUser, authPass); + } + private void expectAuthenticationExceptionForInvalidPassword() { thrown.expect(AuthenticationException.class); thrown.expectMessage("a null or blank password has been provided"); diff --git service/src/test/resources/creds/test.jceks service/src/test/resources/creds/test.jceks new file mode 100755 index 0000000000000000000000000000000000000000..8d58c414192d9a4a6d1f48ce876057db7a1210f4 GIT binary patch literal 534 zcmX?i?%X*B1_mY|W&~np2K9{0vQ)j|)S|M~A|t)T(vpnSypqi1#FEVXJiVNh!~(sf z%)FF>a-jN*o$JJ7*1Tn4EiPg(PR`HOD=y8`OD?J`D9P6=D9SI(Oi3-$3rx#zfQ?O`LImG{?Pof4PD#r`12UaX@bCD`J>%chg_ zgPM)cean7ax$pVHml2X57x_NFyDU(0LgdMn1{GV9_J7xY-x1gT{BeO-fSO4q!}a4U zUloVX-lHOOc2(ze@e2vpT)Np!OBlogoLs{*OEP?2Oda!5LW(jAa#CGfgK