diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index bd884a9234..672724b95c 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -535,13 +535,6 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal + "with the hive data and metadata replication. Set the configuration " + "hive.repl.include.authorization.metadata to false to disable " + "security policies being replicated "), - REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT("hive.repl.authorization.provider.service.endpoint", - "", - "This configuration will define the authorization service endpoint"), - REPL_RANGER_SERVICE_NAME("hive.repl.ranger.service.name", - "hive", - "This configuration will define the service name for which the ranger authorization" - + " policies needs to be replicated"), REPL_RANGER_ADD_DENY_POLICY_TARGET("hive.repl.ranger.target.deny.policy", true, "This configuration will add a deny policy on the target database for all users except hive" diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index eeb81dad25..2278ec0aa7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -1501,8 +1501,7 @@ public Boolean apply(@Nullable CallerArguments args) { @Test public void testRangerReplication() throws Throwable { List clause = Arrays.asList("'hive.repl.include.authorization.metadata'='true'", - "'hive.in.test'='true'", - "'hive.repl.authorization.provider.service.endpoint'='http://localhost:6080/ranger'"); + "'hive.in.test'='true'"); primary.run("use " + primaryDbName) .run("create table acid_table (key int, value int) partitioned by (load_date date) " + "clustered by(key) into 2 buckets stored as orc tblproperties ('transactional'='true')") @@ -1536,8 +1535,7 @@ public void testFailureRangerReplication() throws Throwable { try { primary.dump(primaryDbName, clause); } catch (Exception e) { - assertEquals("Ranger endpoint is not valid. Please pass a valid config " - + "hive.repl.authorization.provider.service.endpoint", e.getMessage()); + assertEquals("Ranger endpoint is not valid.", e.getMessage()); } } diff --git a/itests/hive-unit/src/test/resources/ranger-hive-security.xml b/itests/hive-unit/src/test/resources/ranger-hive-security.xml new file mode 100644 index 0000000000..0e6db83194 --- /dev/null +++ b/itests/hive-unit/src/test/resources/ranger-hive-security.xml @@ -0,0 +1,31 @@ + + + + + + + + + ranger.plugin.hive.service.name + hive + + + ranger.plugin.hive.policy.rest.url + https://ranger.apache.org:5088 + + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerDumpTask.java index e0783712fe..904a431ae6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerDumpTask.java @@ -36,12 +36,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.InputStream; import java.io.Serializable; import java.util.List; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_RANGER_SERVICE_NAME; - /** * RangerDumpTask. * @@ -81,12 +79,16 @@ public int execute() { if (rangerRestClient == null) { rangerRestClient = getRangerRestClient(); } - String rangerEndpoint = conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT); + InputStream inputStream = RangerDumpTask.class.getClassLoader() + .getResourceAsStream(ReplUtils.RANGER_CONFIGURATION_RESOURCE_NAME); + if (inputStream != null) { + conf.addResource(inputStream); + } + String rangerHiveServiceName = conf.get(ReplUtils.RANGER_HIVE_SERVICE_NAME); + String rangerEndpoint = conf.get(ReplUtils.RANGER_REST_URL); if (StringUtils.isEmpty(rangerEndpoint) || !rangerRestClient.checkConnection(rangerEndpoint)) { - throw new Exception("Ranger endpoint is not valid. " - + "Please pass a valid config hive.repl.authorization.provider.service.endpoint"); + throw new Exception("Ranger endpoint is not valid."); } - String rangerHiveServiceName = conf.getVar(REPL_RANGER_SERVICE_NAME); replLogger = new RangerDumpLogger(work.getDbName(), work.getCurrentDumpPath().toString()); replLogger.startLog(); RangerExportPolicyList rangerExportPolicyList = rangerRestClient.exportRangerPolicies(rangerEndpoint, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java index 2c216fff1b..b4f8494619 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java @@ -36,14 +36,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.InputStream; import java.io.Serializable; import java.util.ArrayList; import java.util.List; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_RANGER_ADD_DENY_POLICY_TARGET; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_RANGER_SERVICE_NAME; - /** * RangerLoadTask. * @@ -83,10 +81,15 @@ public int execute() { if (rangerRestClient == null) { rangerRestClient = getRangerRestClient(); } - String rangerEndpoint = conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT); + InputStream inputStream = RangerDumpTask.class.getClassLoader() + .getResourceAsStream(ReplUtils.RANGER_CONFIGURATION_RESOURCE_NAME); + if (inputStream != null) { + conf.addResource(inputStream); + } + String rangerHiveServiceName = conf.get(ReplUtils.RANGER_HIVE_SERVICE_NAME); + String rangerEndpoint = conf.get(ReplUtils.RANGER_REST_URL); if (StringUtils.isEmpty(rangerEndpoint) || !rangerRestClient.checkConnection(rangerEndpoint)) { - throw new Exception("Ranger endpoint is not valid. " - + "Please pass a valid config hive.repl.authorization.provider.service.endpoint"); + throw new Exception("Ranger endpoint is not valid."); } if (work.getCurrentDumpPath() != null) { LOG.info("Importing Ranger Metadata from {} ", work.getCurrentDumpPath()); @@ -108,7 +111,7 @@ public int execute() { List rangerPoliciesWithDenyPolicy = rangerPolicies; if (conf.getBoolVar(REPL_RANGER_ADD_DENY_POLICY_TARGET)) { rangerPoliciesWithDenyPolicy = rangerRestClient.addDenyPolicies(rangerPolicies, - conf.getVar(REPL_RANGER_SERVICE_NAME), work.getSourceDbName(), work.getTargetDbName()); + rangerHiveServiceName, work.getSourceDbName(), work.getTargetDbName()); } List updatedRangerPolicies = rangerRestClient.changeDataSet(rangerPoliciesWithDenyPolicy, @@ -121,7 +124,7 @@ public int execute() { } rangerExportPolicyList.setPolicies(updatedRangerPolicies); rangerRestClient.importRangerPolicies(rangerExportPolicyList, work.getTargetDbName(), rangerEndpoint, - conf.getVar(REPL_RANGER_SERVICE_NAME)); + rangerHiveServiceName); LOG.info("Number of ranger policies imported {}", rangerExportPolicyList.getListSize()); importCount = rangerExportPolicyList.getListSize(); replLogger.endLog(importCount); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java index 377f742a70..543ceca3e4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java @@ -107,6 +107,12 @@ public static final String RANGER_AUTHORIZER = "ranger"; public static final String HIVE_RANGER_POLICIES_FILE_NAME = "ranger_policies.json"; + + public static final String RANGER_REST_URL = "ranger.plugin.hive.policy.rest.url"; + + public static final String RANGER_HIVE_SERVICE_NAME = "ranger.plugin.hive.service.name"; + + public static final String RANGER_CONFIGURATION_RESOURCE_NAME = "ranger-hive-security.xml"; /** * Bootstrap REPL LOAD operation type on the examined object based on ckpt state. */ diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerDumpTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerDumpTask.java index 8ef09876af..e65b8bfdad 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerDumpTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerDumpTask.java @@ -41,8 +41,8 @@ import java.util.ArrayList; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_RANGER_SERVICE_NAME; +import static org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.RANGER_REST_URL; +import static org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.RANGER_HIVE_SERVICE_NAME; /** * Unit test class for testing Ranger Dump. @@ -71,7 +71,7 @@ public void setup() throws Exception { @Test public void testFailureInvalidAuthProviderEndpoint() throws Exception { - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn(null); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn(null); int status = task.execute(); Assert.assertEquals(40000, status); } @@ -82,8 +82,8 @@ public void testSuccessValidAuthProviderEndpoint() throws Exception { rangerPolicyList.setPolicies(new ArrayList()); Mockito.when(mockClient.exportRangerPolicies(Mockito.anyString(), Mockito.anyString(), Mockito.anyString())) .thenReturn(rangerPolicyList); - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); - Mockito.when(conf.getVar(REPL_RANGER_SERVICE_NAME)).thenReturn("hive"); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn("rangerEndpoint"); + Mockito.when(conf.get(RANGER_HIVE_SERVICE_NAME)).thenReturn("hive"); Mockito.when(work.getDbName()).thenReturn("testdb"); Mockito.when(work.getCurrentDumpPath()).thenReturn(new Path("/tmp")); int status = task.execute(); @@ -106,8 +106,8 @@ public void testSuccessNonEmptyRangerPolicies() throws Exception { RangerExportPolicyList rangerPolicyList = new Gson().fromJson(rangerResponse, RangerExportPolicyList.class); Mockito.when(mockClient.exportRangerPolicies(Mockito.anyString(), Mockito.anyString(), Mockito.anyString())) .thenReturn(rangerPolicyList); - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); - Mockito.when(conf.getVar(REPL_RANGER_SERVICE_NAME)).thenReturn("hive"); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn("rangerEndpoint"); + Mockito.when(conf.get(RANGER_HIVE_SERVICE_NAME)).thenReturn("hive"); Mockito.when(work.getDbName()).thenReturn("testdb"); Path rangerDumpPath = new Path("/tmp"); Mockito.when(work.getCurrentDumpPath()).thenReturn(rangerDumpPath); @@ -126,8 +126,8 @@ public void testSuccessRangerDumpMetrics() throws Exception { rangerPolicyList.setPolicies(new ArrayList()); Mockito.when(mockClient.exportRangerPolicies(Mockito.anyString(), Mockito.anyString(), Mockito.anyString())) .thenReturn(rangerPolicyList); - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); - Mockito.when(conf.getVar(REPL_RANGER_SERVICE_NAME)).thenReturn("hive"); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn("rangerEndpoint"); + Mockito.when(conf.get(RANGER_HIVE_SERVICE_NAME)).thenReturn("hive"); Mockito.when(work.getDbName()).thenReturn("testdb"); Mockito.when(work.getCurrentDumpPath()).thenReturn(new Path("/tmp")); int status = task.execute(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java index 8ff2961a61..33c0f2ec03 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java @@ -36,9 +36,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_RANGER_SERVICE_NAME; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_RANGER_ADD_DENY_POLICY_TARGET; +import static org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.RANGER_HIVE_SERVICE_NAME; +import static org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.RANGER_REST_URL; /** * Unit test class for testing Ranger Dump. @@ -70,14 +70,14 @@ public void setup() throws Exception { @Test public void testFailureInvalidAuthProviderEndpoint() { - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn(null); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn(null); int status = task.execute(); Assert.assertEquals(40000, status); } @Test public void testSuccessValidAuthProviderEndpoint() { - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn("rangerEndpoint"); Mockito.when(work.getSourceDbName()).thenReturn("srcdb"); Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); int status = task.execute(); @@ -98,7 +98,7 @@ public void testSuccessNonEmptyRangerPolicies() throws Exception { + "\"dataMaskPolicyItems\":[],\"rowFilterPolicyItems\":[],\"id\":40,\"guid\":" + "\"4e2b3406-7b9a-4004-8cdf-7a239c8e2cae\",\"isEnabled\":true,\"version\":1}]}"; RangerExportPolicyList rangerPolicyList = new Gson().fromJson(rangerResponse, RangerExportPolicyList.class); - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn("rangerEndpoint"); Mockito.when(work.getSourceDbName()).thenReturn("srcdb"); Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); Path rangerDumpPath = new Path("/tmp"); @@ -124,7 +124,7 @@ public void testSuccessRangerDumpMetrics() throws Exception { + "\"dataMaskPolicyItems\":[],\"rowFilterPolicyItems\":[],\"id\":40,\"guid\":" + "\"4e2b3406-7b9a-4004-8cdf-7a239c8e2cae\",\"isEnabled\":true,\"version\":1}]}"; RangerExportPolicyList rangerPolicyList = new Gson().fromJson(rangerResponse, RangerExportPolicyList.class); - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn("rangerEndpoint"); Mockito.when(work.getSourceDbName()).thenReturn("srcdb"); Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); Path rangerDumpPath = new Path("/tmp"); @@ -163,10 +163,10 @@ public void testSuccessAddDenyRangerPolicies() throws Exception { + "\"dataMaskPolicyItems\":[],\"rowFilterPolicyItems\":[],\"id\":40,\"guid\":" + "\"4e2b3406-7b9a-4004-8cdf-7a239c8e2cae\",\"isEnabled\":true,\"version\":1}]}"; RangerExportPolicyList rangerPolicyList = new Gson().fromJson(rangerResponse, RangerExportPolicyList.class); - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn("rangerEndpoint"); Mockito.when(work.getSourceDbName()).thenReturn("srcdb"); Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); - Mockito.when(conf.getVar(REPL_RANGER_SERVICE_NAME)).thenReturn("hive"); + Mockito.when(conf.get(RANGER_HIVE_SERVICE_NAME)).thenReturn("hive"); Mockito.when(conf.getBoolVar(REPL_RANGER_ADD_DENY_POLICY_TARGET)).thenReturn(true); Path rangerDumpPath = new Path("/tmp"); Mockito.when(work.getCurrentDumpPath()).thenReturn(rangerDumpPath); @@ -226,10 +226,10 @@ public void testSuccessDisableDenyRangerPolicies() throws Exception { + "\"dataMaskPolicyItems\":[],\"rowFilterPolicyItems\":[],\"id\":40,\"guid\":" + "\"4e2b3406-7b9a-4004-8cdf-7a239c8e2cae\",\"isEnabled\":true,\"version\":1}]}"; RangerExportPolicyList rangerPolicyList = new Gson().fromJson(rangerResponse, RangerExportPolicyList.class); - Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); + Mockito.when(conf.get(RANGER_REST_URL)).thenReturn("rangerEndpoint"); Mockito.when(work.getSourceDbName()).thenReturn("srcdb"); Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); - Mockito.when(conf.getVar(REPL_RANGER_SERVICE_NAME)).thenReturn("hive"); + Mockito.when(conf.get(RANGER_HIVE_SERVICE_NAME)).thenReturn("hive"); Mockito.when(conf.getBoolVar(REPL_RANGER_ADD_DENY_POLICY_TARGET)).thenReturn(false); Path rangerDumpPath = new Path("/tmp"); Mockito.when(work.getCurrentDumpPath()).thenReturn(rangerDumpPath);