diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java index 5497d28eff..31d9979d40 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java @@ -96,8 +96,12 @@ public int execute() { LOG.info("There are no ranger policies to import"); rangerPolicies = new ArrayList<>(); } - List updatedRangerPolicies = rangerRestClient.changeDataSet(rangerPolicies, work.getSourceDbName(), - work.getTargetDbName()); + List rangerPoliciesWithDenyPolicy = rangerRestClient.addDenyPolicies(rangerPolicies, + conf.getVar(REPL_RANGER_SERVICE_NAME), work.getSourceDbName(), work.getTargetDbName()); + + List updatedRangerPolicies = rangerRestClient.changeDataSet(rangerPoliciesWithDenyPolicy, + work.getSourceDbName(), work.getTargetDbName()); + int importCount = 0; if (!CollectionUtils.isEmpty(updatedRangerPolicies)) { if (rangerExportPolicyList == null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/NoOpRangerRestClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/NoOpRangerRestClient.java index 4e3fa61c42..95b1f3ac50 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/NoOpRangerRestClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/NoOpRangerRestClient.java @@ -68,4 +68,11 @@ public RangerExportPolicyList readRangerPoliciesFromJsonFile(Path filePath, Hive public boolean checkConnection(String url) throws Exception { return true; } + + @Override + public List addDenyPolicies(List rangerPolicies, String rangerServiceName, + String sourceDb, String targetDb) { + return rangerPolicies; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClient.java index eab20f459e..bb3011cd54 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClient.java @@ -47,4 +47,7 @@ RangerExportPolicyList readRangerPoliciesFromJsonFile(Path filePath, HiveConf conf) throws SemanticException; boolean checkConnection(String url) throws Exception; + + List addDenyPolicies(List rangerPolicies, String rangerServiceName, + String sourceDb, String targetDb); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClientImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClientImpl.java index c535f9ea07..e068e1ec2e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClientImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClientImpl.java @@ -49,12 +49,14 @@ import java.io.InputStreamReader; import java.io.InputStream; import java.io.Reader; +import java.io.FileNotFoundException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.ArrayList; +import java.util.HashMap; /** * RangerRestClientImpl to connect to Ranger and export policies. @@ -335,6 +337,9 @@ public RangerExportPolicyList readRangerPoliciesFromJsonFile(Path filePath, InputStream inputStream = fs.open(filePath); Reader reader = new InputStreamReader(inputStream, Charset.forName("UTF-8")); rangerExportPolicyList = gsonBuilder.fromJson(reader, RangerExportPolicyList.class); + } catch (FileNotFoundException e) { + //If the ranger policies are not present, json file will not be present + return rangerExportPolicyList; } catch (Exception ex) { throw new SemanticException("Error reading file :" + filePath, ex); } @@ -349,6 +354,93 @@ public boolean checkConnection(String url) { return (clientResp.getStatus() < HttpServletResponse.SC_UNAUTHORIZED); } + @Override + public List addDenyPolicies(List rangerPolicies, String rangerServiceName, + String sourceDb, String targetDb) { + List rangerPoliciesToImport = new ArrayList(); + if (CollectionUtils.isNotEmpty(rangerPolicies)) { + for (RangerPolicy rangerPolicy : rangerPolicies) { + rangerPoliciesToImport.add(rangerPolicy); + } + } + + RangerPolicy denyRangerPolicy = null; + + if (sourceDb.endsWith("/")) { + sourceDb = StringUtils.removePattern(sourceDb, "/+$"); + } + if (targetDb.endsWith("/")) { + targetDb = StringUtils.removePattern(targetDb, "/+$"); + } + if (!StringUtils.isEmpty(rangerServiceName)) { + denyRangerPolicy = new RangerPolicy(); + denyRangerPolicy.setService(rangerServiceName); + denyRangerPolicy.setName(sourceDb + "_replication deny policy for " + targetDb); + } + if (denyRangerPolicy != null) { + Map rangerPolicyResourceMap = + new HashMap(); + RangerPolicy.RangerPolicyResource rangerPolicyResource = new RangerPolicy.RangerPolicyResource(); + List resourceNameList = new ArrayList(); + + List denyPolicyItemsForPublicGroup = denyRangerPolicy.getDenyPolicyItems(); + RangerPolicy.RangerPolicyItem denyPolicyItem = new RangerPolicy.RangerPolicyItem(); + List denyPolicyItemAccesses = + new ArrayList(); + + List denyExceptionsItemsForBeaconUser = + denyRangerPolicy.getDenyExceptions(); + RangerPolicy.RangerPolicyItem denyExceptionsPolicyItem = new RangerPolicy.RangerPolicyItem(); + List denyExceptionsPolicyItemAccesses = + new ArrayList(); + + resourceNameList.add(sourceDb); + rangerPolicyResource.setValues(resourceNameList); + RangerPolicy.RangerPolicyResource rangerPolicyResourceColumn =new RangerPolicy.RangerPolicyResource(); + rangerPolicyResourceColumn.setValues(new ArrayList(){{add("*"); }}); + RangerPolicy.RangerPolicyResource rangerPolicyResourceTable =new RangerPolicy.RangerPolicyResource(); + rangerPolicyResourceTable.setValues(new ArrayList(){{add("*"); }}); + rangerPolicyResourceMap.put("database", rangerPolicyResource); + rangerPolicyResourceMap.put("table", rangerPolicyResourceTable); + rangerPolicyResourceMap.put("column", rangerPolicyResourceColumn); + denyRangerPolicy.setResources(rangerPolicyResourceMap); + + denyPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("create", true)); + denyPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("update", true)); + denyPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("drop", true)); + denyPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("alter", true)); + denyPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("index", true)); + denyPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("lock", true)); + denyPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("write", true)); + denyPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("ReplAdmin", true)); + denyPolicyItem.setAccesses(denyPolicyItemAccesses); + denyPolicyItemsForPublicGroup.add(denyPolicyItem); + List denyPolicyItemsGroups = new ArrayList(); + denyPolicyItemsGroups.add("public"); + denyPolicyItem.setGroups(denyPolicyItemsGroups); + denyRangerPolicy.setDenyPolicyItems(denyPolicyItemsForPublicGroup); + + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("create", true)); + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("update", true)); + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("drop", true)); + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("alter", true)); + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("index", true)); + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("lock", true)); + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("write", true)); + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("select", true)); + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("read", true)); + denyExceptionsPolicyItem.setAccesses(denyExceptionsPolicyItemAccesses); + denyExceptionsItemsForBeaconUser.add(denyExceptionsPolicyItem); + List denyExceptionsPolicyItemsUsers = new ArrayList(); + denyExceptionsPolicyItemsUsers.add("hive"); + denyExceptionsPolicyItem.setUsers(denyExceptionsPolicyItemsUsers); + denyRangerPolicy.setDenyExceptions(denyExceptionsItemsForBeaconUser); + + rangerPoliciesToImport.add(denyRangerPolicy); + } + return rangerPoliciesToImport; + } + private WebResource.Builder getRangerResourceBuilder(String url) { Client client = getRangerClient(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java index 10c1afd2b2..bf3ef0adf4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java @@ -21,12 +21,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.repl.ranger.RangerExportPolicyList; +import org.apache.hadoop.hive.ql.exec.repl.ranger.RangerPolicy; import org.apache.hadoop.hive.ql.exec.repl.ranger.RangerRestClientImpl; import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @@ -35,6 +37,7 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_RANGER_SERVICE_NAME; /** * Unit test class for testing Ranger Dump. @@ -59,6 +62,8 @@ public void setup() throws Exception { task = new RangerLoadTask(mockClient, conf, work); Mockito.when(mockClient.changeDataSet(Mockito.anyList(), Mockito.anyString(), Mockito.anyString())) .thenCallRealMethod(); + Mockito.when(mockClient.addDenyPolicies(Mockito.anyList(), Mockito.anyString(), Mockito.anyString(), + Mockito.anyString())).thenCallRealMethod(); Mockito.when(mockClient.checkConnection(Mockito.anyString())).thenReturn(true); } @@ -82,7 +87,7 @@ public void testSuccessValidAuthProviderEndpoint() { public void testSuccessNonEmptyRangerPolicies() throws Exception { String rangerResponse = "{\"metaDataInfo\":{\"Host name\":\"ranger.apache.org\"," + "\"Exported by\":\"hive\",\"Export time\":\"May 5, 2020, 8:55:03 AM\",\"Ranger apache version\"" - + ":\"2.0.0.7.2.0.0-61\"},\"policies\":[{\"service\":\"cm_hive\",\"name\":\"db-level\",\"policyType\":0," + + ":\"2.0.0.7.2.0.0-61\"},\"policies\":[{\"service\":\"hive\",\"name\":\"db-level\",\"policyType\":0," + "\"description\":\"\",\"isAuditEnabled\":true,\"resources\":{\"database\":{\"values\":[\"aa\"]," + "\"isExcludes\":false,\"isRecursive\":false},\"column\":{\"values\":[\"id\"],\"isExcludes\":false," + "\"isRecursive\":false},\"table\":{\"values\":[\"*\"],\"isExcludes\":false,\"isRecursive\":false}}," @@ -103,4 +108,72 @@ public void testSuccessNonEmptyRangerPolicies() throws Exception { int status = task.execute(); Assert.assertEquals(0, status); } + + @Test + public void testSuccessAddDenyRangerPolicies() throws Exception { + String rangerResponse = "{\"metaDataInfo\":{\"Host name\":\"ranger.apache.org\"," + + "\"Exported by\":\"hive\",\"Export time\":\"May 5, 2020, 8:55:03 AM\",\"Ranger apache version\"" + + ":\"2.0.0.7.2.0.0-61\"},\"policies\":[{\"service\":\"hive\",\"name\":\"db-level\",\"policyType\":0," + + "\"description\":\"\",\"isAuditEnabled\":true,\"resources\":{\"database\":{\"values\":[\"aa\"]," + + "\"isExcludes\":false,\"isRecursive\":false},\"column\":{\"values\":[\"id\"],\"isExcludes\":false," + + "\"isRecursive\":false},\"table\":{\"values\":[\"*\"],\"isExcludes\":false,\"isRecursive\":false}}," + + "\"policyItems\":[{\"accesses\":[{\"type\":\"select\",\"isAllowed\":true},{\"type\":\"update\"," + + "\"isAllowed\":true}],\"users\":[\"admin\"],\"groups\":[\"public\"],\"conditions\":[]," + + "\"delegateAdmin\":false}],\"denyPolicyItems\":[],\"allowExceptions\":[],\"denyExceptions\":[]," + + "\"dataMaskPolicyItems\":[],\"rowFilterPolicyItems\":[],\"id\":40,\"guid\":" + + "\"4e2b3406-7b9a-4004-8cdf-7a239c8e2cae\",\"isEnabled\":true,\"version\":1}]}"; + RangerExportPolicyList rangerPolicyList = new Gson().fromJson(rangerResponse, RangerExportPolicyList.class); + Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); + Mockito.when(work.getSourceDbName()).thenReturn("srcdb"); + Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); + Mockito.when(conf.getVar(REPL_RANGER_SERVICE_NAME)).thenReturn("hive"); + Path rangerDumpPath = new Path("/tmp"); + Mockito.when(work.getCurrentDumpPath()).thenReturn(rangerDumpPath); + mockClient.saveRangerPoliciesToFile(rangerPolicyList, + rangerDumpPath, ReplUtils.HIVE_RANGER_POLICIES_FILE_NAME, new HiveConf()); + Mockito.when(mockClient.readRangerPoliciesFromJsonFile(Mockito.any(), Mockito.any())).thenReturn(rangerPolicyList); + int status = task.execute(); + Assert.assertEquals(0, status); + + + + + ArgumentCaptor rangerPolicyCapture = ArgumentCaptor.forClass(RangerExportPolicyList.class); + ArgumentCaptor rangerEndpoint = ArgumentCaptor.forClass(String.class); + ArgumentCaptor serviceName = ArgumentCaptor.forClass(String.class); + ArgumentCaptor targetDb = ArgumentCaptor.forClass(String.class); + Mockito.verify(mockClient, + Mockito.times(1)).importRangerPolicies(rangerPolicyCapture.capture(), + targetDb.capture(), rangerEndpoint.capture(), serviceName.capture()); + Assert.assertEquals("tgtdb", targetDb.getAllValues().get(0)); + Assert.assertEquals("rangerEndpoint", rangerEndpoint.getAllValues().get(0)); + Assert.assertEquals("hive", serviceName.getAllValues().get(0)); + RangerExportPolicyList actualPolicyList = rangerPolicyCapture.getAllValues().get(0); + Assert.assertEquals(rangerPolicyList.getMetaDataInfo(), actualPolicyList.getMetaDataInfo()); + //Deny policy is added + Assert.assertEquals(2, actualPolicyList.getListSize()); + RangerPolicy denyPolicy = actualPolicyList.getPolicies().get(1); + Assert.assertEquals("hive", denyPolicy.getService()); + Assert.assertEquals("srcdb_replication deny policy for tgtdb", denyPolicy.getName()); + Assert.assertEquals(1, denyPolicy.getDenyExceptions().size()); + Assert.assertEquals("public", denyPolicy.getDenyPolicyItems().get(0).getGroups().get(0)); + Assert.assertEquals(8, denyPolicy.getDenyPolicyItems().get(0).getAccesses().size()); + boolean isReplAdminDenied = false; + for (RangerPolicy.RangerPolicyItemAccess access : denyPolicy.getDenyPolicyItems().get(0).getAccesses()) { + if (access.getType().equalsIgnoreCase("ReplAdmin")) { + isReplAdminDenied = true; + } + } + Assert.assertTrue(isReplAdminDenied); + //Deny exception is for hive user. Deny exception is not for repl admin permission + Assert.assertEquals("hive", denyPolicy.getDenyExceptions().get(0).getUsers().get(0)); + Assert.assertEquals(9, denyPolicy.getDenyExceptions().get(0).getAccesses().size()); + isReplAdminDenied = false; + for (RangerPolicy.RangerPolicyItemAccess access : denyPolicy.getDenyExceptions().get(0).getAccesses()) { + if (access.getType().equalsIgnoreCase("ReplAdmin")) { + isReplAdminDenied = true; + } + } + Assert.assertFalse(isReplAdminDenied); + } }