diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index f5ad3a882b..0c536b8314 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -542,6 +542,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "hive", "This configuration will define the service name for which the ranger authorization" + " policies needs to be replicated"), + REPL_RANGER_ADD_DENY_POLICY_TARGET("hive.repl.ranger.target.deny.policy", + true, + "This configuration will add a deny policy on the target database for all users except hive" + + " to avoid any update to the target database"), LOCALSCRATCHDIR("hive.exec.local.scratchdir", "${system:java.io.tmpdir}" + File.separator + "${system:user.name}", "Local scratch space for Hive jobs"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java index 4d62a512cd..4580422e3e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/RangerLoadTask.java @@ -40,8 +40,7 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_RANGER_SERVICE_NAME; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.*; /** * RangerLoadTask. @@ -104,8 +103,15 @@ public int execute() { LOG.info("There are no ranger policies to import"); rangerPolicies = new ArrayList<>(); } - List updatedRangerPolicies = rangerRestClient.changeDataSet(rangerPolicies, work.getSourceDbName(), - work.getTargetDbName()); + List rangerPoliciesWithDenyPolicy = rangerPolicies; + if (conf.getBoolVar(REPL_RANGER_ADD_DENY_POLICY_TARGET)) { + rangerPoliciesWithDenyPolicy = rangerRestClient.addDenyPolicies(rangerPolicies, + conf.getVar(REPL_RANGER_SERVICE_NAME), work.getSourceDbName(), work.getTargetDbName()); + } + + List updatedRangerPolicies = rangerRestClient.changeDataSet(rangerPoliciesWithDenyPolicy, + work.getSourceDbName(), work.getTargetDbName()); + long importCount = 0; if (!CollectionUtils.isEmpty(updatedRangerPolicies)) { if (rangerExportPolicyList == null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/NoOpRangerRestClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/NoOpRangerRestClient.java index b0fdff434f..5a11430544 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/NoOpRangerRestClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/NoOpRangerRestClient.java @@ -68,4 +68,11 @@ public RangerExportPolicyList readRangerPoliciesFromJsonFile(Path filePath, Hive public boolean checkConnection(String url) throws Exception { return true; } + + @Override + public List addDenyPolicies(List rangerPolicies, String rangerServiceName, + String sourceDb, String targetDb) throws SemanticException { + return rangerPolicies; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClient.java index eab20f459e..f85c3afbd3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClient.java @@ -47,4 +47,7 @@ RangerExportPolicyList readRangerPoliciesFromJsonFile(Path filePath, HiveConf conf) throws SemanticException; boolean checkConnection(String url) throws Exception; + + List addDenyPolicies(List rangerPolicies, String rangerServiceName, + String sourceDb, String targetDb) throws SemanticException; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClientImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClientImpl.java index c535f9ea07..2ef90cfb09 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClientImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ranger/RangerRestClientImpl.java @@ -49,12 +49,10 @@ import java.io.InputStreamReader; import java.io.InputStream; import java.io.Reader; +import java.io.FileNotFoundException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.ArrayList; +import java.util.*; /** * RangerRestClientImpl to connect to Ranger and export policies. @@ -225,12 +223,6 @@ private synchronized Client getRangerClient() { @Override public List changeDataSet(List rangerPolicies, String sourceDbName, String targetDbName) { - if (sourceDbName.endsWith("/")) { - sourceDbName = StringUtils.removePattern(sourceDbName, "/+$"); - } - if (targetDbName.endsWith("/")) { - targetDbName = StringUtils.removePattern(targetDbName, "/+$"); - } if (targetDbName.equals(sourceDbName)) { return rangerPolicies; } @@ -335,6 +327,9 @@ public RangerExportPolicyList readRangerPoliciesFromJsonFile(Path filePath, InputStream inputStream = fs.open(filePath); Reader reader = new InputStreamReader(inputStream, Charset.forName("UTF-8")); rangerExportPolicyList = gsonBuilder.fromJson(reader, RangerExportPolicyList.class); + } catch (FileNotFoundException e) { + //If the ranger policies are not present, json file will not be present + return rangerExportPolicyList; } catch (Exception ex) { throw new SemanticException("Error reading file :" + filePath, ex); } @@ -349,6 +344,70 @@ public boolean checkConnection(String url) { return (clientResp.getStatus() < HttpServletResponse.SC_UNAUTHORIZED); } + @Override + public List addDenyPolicies(List rangerPolicies, String rangerServiceName, + String sourceDb, String targetDb) throws SemanticException { + if (StringUtils.isEmpty(rangerServiceName)) { + throw new SemanticException("Ranger Service Name cannot be empty"); + } + RangerPolicy denyRangerPolicy = new RangerPolicy(); + denyRangerPolicy.setService(rangerServiceName); + denyRangerPolicy.setName(sourceDb + "_replication deny policy for " + targetDb); + Map rangerPolicyResourceMap = new HashMap(); + RangerPolicy.RangerPolicyResource rangerPolicyResource = new RangerPolicy.RangerPolicyResource(); + List resourceNameList = new ArrayList(); + + List denyPolicyItemsForPublicGroup = denyRangerPolicy.getDenyPolicyItems(); + RangerPolicy.RangerPolicyItem denyPolicyItem = new RangerPolicy.RangerPolicyItem(); + List denyPolicyItemAccesses = new ArrayList(); + + List denyExceptionsItemsForBeaconUser = denyRangerPolicy.getDenyExceptions(); + RangerPolicy.RangerPolicyItem denyExceptionsPolicyItem = new RangerPolicy.RangerPolicyItem(); + List denyExceptionsPolicyItemAccesses = new ArrayList(); + + resourceNameList.add(sourceDb); + rangerPolicyResource.setValues(resourceNameList); + RangerPolicy.RangerPolicyResource rangerPolicyResourceColumn =new RangerPolicy.RangerPolicyResource(); + rangerPolicyResourceColumn.setValues(new ArrayList(){{add("*"); }}); + RangerPolicy.RangerPolicyResource rangerPolicyResourceTable =new RangerPolicy.RangerPolicyResource(); + rangerPolicyResourceTable.setValues(new ArrayList(){{add("*"); }}); + rangerPolicyResourceMap.put("database", rangerPolicyResource); + rangerPolicyResourceMap.put("table", rangerPolicyResourceTable); + rangerPolicyResourceMap.put("column", rangerPolicyResourceColumn); + denyRangerPolicy.setResources(rangerPolicyResourceMap); + + List accessTypes = new ArrayList<>(); + accessTypes.addAll(Arrays.asList("create", "update", "drop", "alter", "index", "lock", "write", "ReplAdmin")); + for (String access : accessTypes) { + denyPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess(access, true)); + } + denyPolicyItem.setAccesses(denyPolicyItemAccesses); + denyPolicyItemsForPublicGroup.add(denyPolicyItem); + List denyPolicyItemsGroups = new ArrayList(); + denyPolicyItemsGroups.add("public"); + denyPolicyItem.setGroups(denyPolicyItemsGroups); + denyRangerPolicy.setDenyPolicyItems(denyPolicyItemsForPublicGroup); + + List denyExcludeAccessTypes = new ArrayList<>(); + denyExcludeAccessTypes.addAll(Arrays.asList("create", "update", "drop", "alter", "index", "lock", "write", + "ReplAdmin", "select", "read")); + for (String access : denyExcludeAccessTypes) { + denyExceptionsPolicyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess(access, true)); + } + denyExceptionsPolicyItem.setAccesses(denyExceptionsPolicyItemAccesses); + denyExceptionsItemsForBeaconUser.add(denyExceptionsPolicyItem); + List denyExceptionsPolicyItemsUsers = new ArrayList(); + denyExceptionsPolicyItemsUsers.add("hive"); + denyExceptionsPolicyItem.setUsers(denyExceptionsPolicyItemsUsers); + denyRangerPolicy.setDenyExceptions(denyExceptionsItemsForBeaconUser); + + rangerPolicies.add(denyRangerPolicy); + return rangerPolicies; + } + private WebResource.Builder getRangerResourceBuilder(String url) { Client client = getRangerClient(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java index 296bd3875a..c2cddc9e56 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestRangerLoadTask.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.repl.ranger.RangerExportPolicyList; +import org.apache.hadoop.hive.ql.exec.repl.ranger.RangerPolicy; import org.apache.hadoop.hive.ql.exec.repl.ranger.RangerRestClientImpl; import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils; import org.apache.hadoop.hive.ql.parse.repl.ReplState; @@ -36,7 +37,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.*; /** * Unit test class for testing Ranger Dump. @@ -61,6 +62,8 @@ public void setup() throws Exception { task = new RangerLoadTask(mockClient, conf, work); Mockito.when(mockClient.changeDataSet(Mockito.anyList(), Mockito.anyString(), Mockito.anyString())) .thenCallRealMethod(); + Mockito.when(mockClient.addDenyPolicies(Mockito.anyList(), Mockito.anyString(), Mockito.anyString(), + Mockito.anyString())).thenCallRealMethod(); Mockito.when(mockClient.checkConnection(Mockito.anyString())).thenReturn(true); } @@ -84,7 +87,7 @@ public void testSuccessValidAuthProviderEndpoint() { public void testSuccessNonEmptyRangerPolicies() throws Exception { String rangerResponse = "{\"metaDataInfo\":{\"Host name\":\"ranger.apache.org\"," + "\"Exported by\":\"hive\",\"Export time\":\"May 5, 2020, 8:55:03 AM\",\"Ranger apache version\"" - + ":\"2.0.0.7.2.0.0-61\"},\"policies\":[{\"service\":\"cm_hive\",\"name\":\"db-level\",\"policyType\":0," + + ":\"2.0.0.7.2.0.0-61\"},\"policies\":[{\"service\":\"hive\",\"name\":\"db-level\",\"policyType\":0," + "\"description\":\"\",\"isAuditEnabled\":true,\"resources\":{\"database\":{\"values\":[\"aa\"]," + "\"isExcludes\":false,\"isRecursive\":false},\"column\":{\"values\":[\"id\"],\"isExcludes\":false," + "\"isRecursive\":false},\"table\":{\"values\":[\"*\"],\"isExcludes\":false,\"isRecursive\":false}}," @@ -99,8 +102,6 @@ public void testSuccessNonEmptyRangerPolicies() throws Exception { Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); Path rangerDumpPath = new Path("/tmp"); Mockito.when(work.getCurrentDumpPath()).thenReturn(rangerDumpPath); - mockClient.saveRangerPoliciesToFile(rangerPolicyList, - rangerDumpPath, ReplUtils.HIVE_RANGER_POLICIES_FILE_NAME, new HiveConf()); Mockito.when(mockClient.readRangerPoliciesFromJsonFile(Mockito.any(), Mockito.any())).thenReturn(rangerPolicyList); int status = task.execute(); Assert.assertEquals(0, status); @@ -127,8 +128,6 @@ public void testSuccessRangerDumpMetrics() throws Exception { Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); Path rangerDumpPath = new Path("/tmp"); Mockito.when(work.getCurrentDumpPath()).thenReturn(rangerDumpPath); - mockClient.saveRangerPoliciesToFile(rangerPolicyList, - rangerDumpPath, ReplUtils.HIVE_RANGER_POLICIES_FILE_NAME, new HiveConf()); Mockito.when(mockClient.readRangerPoliciesFromJsonFile(Mockito.any(), Mockito.any())).thenReturn(rangerPolicyList); int status = task.execute(); Assert.assertEquals(0, status); @@ -148,4 +147,107 @@ public void testSuccessRangerDumpMetrics() throws Exception { .getAllValues().get(1).toString().contains("{\"sourceDbName\":\"srcdb\",\"targetDbName\"" + ":\"tgtdb\",\"actualNumPolicies\":1,\"loadEndTime\"")); } + + @Test + public void testSuccessAddDenyRangerPolicies() throws Exception { + String rangerResponse = "{\"metaDataInfo\":{\"Host name\":\"ranger.apache.org\"," + + "\"Exported by\":\"hive\",\"Export time\":\"May 5, 2020, 8:55:03 AM\",\"Ranger apache version\"" + + ":\"2.0.0.7.2.0.0-61\"},\"policies\":[{\"service\":\"hive\",\"name\":\"db-level\",\"policyType\":0," + + "\"description\":\"\",\"isAuditEnabled\":true,\"resources\":{\"database\":{\"values\":[\"aa\"]," + + "\"isExcludes\":false,\"isRecursive\":false},\"column\":{\"values\":[\"id\"],\"isExcludes\":false," + + "\"isRecursive\":false},\"table\":{\"values\":[\"*\"],\"isExcludes\":false,\"isRecursive\":false}}," + + "\"policyItems\":[{\"accesses\":[{\"type\":\"select\",\"isAllowed\":true},{\"type\":\"update\"," + + "\"isAllowed\":true}],\"users\":[\"admin\"],\"groups\":[\"public\"],\"conditions\":[]," + + "\"delegateAdmin\":false}],\"denyPolicyItems\":[],\"allowExceptions\":[],\"denyExceptions\":[]," + + "\"dataMaskPolicyItems\":[],\"rowFilterPolicyItems\":[],\"id\":40,\"guid\":" + + "\"4e2b3406-7b9a-4004-8cdf-7a239c8e2cae\",\"isEnabled\":true,\"version\":1}]}"; + RangerExportPolicyList rangerPolicyList = new Gson().fromJson(rangerResponse, RangerExportPolicyList.class); + Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); + Mockito.when(work.getSourceDbName()).thenReturn("srcdb"); + Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); + Mockito.when(conf.getVar(REPL_RANGER_SERVICE_NAME)).thenReturn("hive"); + Mockito.when(conf.getBoolVar(REPL_RANGER_ADD_DENY_POLICY_TARGET)).thenReturn(true); + Path rangerDumpPath = new Path("/tmp"); + Mockito.when(work.getCurrentDumpPath()).thenReturn(rangerDumpPath); + Mockito.when(mockClient.readRangerPoliciesFromJsonFile(Mockito.any(), Mockito.any())).thenReturn(rangerPolicyList); + int status = task.execute(); + Assert.assertEquals(0, status); + ArgumentCaptor rangerPolicyCapture = ArgumentCaptor.forClass(RangerExportPolicyList.class); + ArgumentCaptor rangerEndpoint = ArgumentCaptor.forClass(String.class); + ArgumentCaptor serviceName = ArgumentCaptor.forClass(String.class); + ArgumentCaptor targetDb = ArgumentCaptor.forClass(String.class); + Mockito.verify(mockClient, + Mockito.times(1)).importRangerPolicies(rangerPolicyCapture.capture(), + targetDb.capture(), rangerEndpoint.capture(), serviceName.capture()); + Assert.assertEquals("tgtdb", targetDb.getAllValues().get(0)); + Assert.assertEquals("rangerEndpoint", rangerEndpoint.getAllValues().get(0)); + Assert.assertEquals("hive", serviceName.getAllValues().get(0)); + RangerExportPolicyList actualPolicyList = rangerPolicyCapture.getAllValues().get(0); + Assert.assertEquals(rangerPolicyList.getMetaDataInfo(), actualPolicyList.getMetaDataInfo()); + //Deny policy is added + Assert.assertEquals(2, actualPolicyList.getListSize()); + RangerPolicy denyPolicy = actualPolicyList.getPolicies().get(1); + Assert.assertEquals("hive", denyPolicy.getService()); + Assert.assertEquals("srcdb_replication deny policy for tgtdb", denyPolicy.getName()); + Assert.assertEquals(1, denyPolicy.getDenyExceptions().size()); + Assert.assertEquals("public", denyPolicy.getDenyPolicyItems().get(0).getGroups().get(0)); + Assert.assertEquals(8, denyPolicy.getDenyPolicyItems().get(0).getAccesses().size()); + boolean isReplAdminDenied = false; + for (RangerPolicy.RangerPolicyItemAccess access : denyPolicy.getDenyPolicyItems().get(0).getAccesses()) { + if (access.getType().equalsIgnoreCase("ReplAdmin")) { + isReplAdminDenied = true; + } + } + Assert.assertTrue(isReplAdminDenied); + //Deny exception is for hive user. Deny exception is not for repl admin permission + Assert.assertEquals("hive", denyPolicy.getDenyExceptions().get(0).getUsers().get(0)); + Assert.assertEquals(10, denyPolicy.getDenyExceptions().get(0).getAccesses().size()); + isReplAdminDenied = false; + for (RangerPolicy.RangerPolicyItemAccess access : denyPolicy.getDenyExceptions().get(0).getAccesses()) { + if (access.getType().equalsIgnoreCase("ReplAdmin")) { + isReplAdminDenied = true; + } + } + Assert.assertTrue(isReplAdminDenied); + } + + @Test + public void testSuccessDisableDenyRangerPolicies() throws Exception { + String rangerResponse = "{\"metaDataInfo\":{\"Host name\":\"ranger.apache.org\"," + + "\"Exported by\":\"hive\",\"Export time\":\"May 5, 2020, 8:55:03 AM\",\"Ranger apache version\"" + + ":\"2.0.0.7.2.0.0-61\"},\"policies\":[{\"service\":\"hive\",\"name\":\"db-level\",\"policyType\":0," + + "\"description\":\"\",\"isAuditEnabled\":true,\"resources\":{\"database\":{\"values\":[\"aa\"]," + + "\"isExcludes\":false,\"isRecursive\":false},\"column\":{\"values\":[\"id\"],\"isExcludes\":false," + + "\"isRecursive\":false},\"table\":{\"values\":[\"*\"],\"isExcludes\":false,\"isRecursive\":false}}," + + "\"policyItems\":[{\"accesses\":[{\"type\":\"select\",\"isAllowed\":true},{\"type\":\"update\"," + + "\"isAllowed\":true}],\"users\":[\"admin\"],\"groups\":[\"public\"],\"conditions\":[]," + + "\"delegateAdmin\":false}],\"denyPolicyItems\":[],\"allowExceptions\":[],\"denyExceptions\":[]," + + "\"dataMaskPolicyItems\":[],\"rowFilterPolicyItems\":[],\"id\":40,\"guid\":" + + "\"4e2b3406-7b9a-4004-8cdf-7a239c8e2cae\",\"isEnabled\":true,\"version\":1}]}"; + RangerExportPolicyList rangerPolicyList = new Gson().fromJson(rangerResponse, RangerExportPolicyList.class); + Mockito.when(conf.getVar(REPL_AUTHORIZATION_PROVIDER_SERVICE_ENDPOINT)).thenReturn("rangerEndpoint"); + Mockito.when(work.getSourceDbName()).thenReturn("srcdb"); + Mockito.when(work.getTargetDbName()).thenReturn("tgtdb"); + Mockito.when(conf.getVar(REPL_RANGER_SERVICE_NAME)).thenReturn("hive"); + Mockito.when(conf.getBoolVar(REPL_RANGER_ADD_DENY_POLICY_TARGET)).thenReturn(false); + Path rangerDumpPath = new Path("/tmp"); + Mockito.when(work.getCurrentDumpPath()).thenReturn(rangerDumpPath); + Mockito.when(mockClient.readRangerPoliciesFromJsonFile(Mockito.any(), Mockito.any())).thenReturn(rangerPolicyList); + int status = task.execute(); + Assert.assertEquals(0, status); + ArgumentCaptor rangerPolicyCapture = ArgumentCaptor.forClass(RangerExportPolicyList.class); + ArgumentCaptor rangerEndpoint = ArgumentCaptor.forClass(String.class); + ArgumentCaptor serviceName = ArgumentCaptor.forClass(String.class); + ArgumentCaptor targetDb = ArgumentCaptor.forClass(String.class); + Mockito.verify(mockClient, + Mockito.times(1)).importRangerPolicies(rangerPolicyCapture.capture(), + targetDb.capture(), rangerEndpoint.capture(), serviceName.capture()); + Assert.assertEquals("tgtdb", targetDb.getAllValues().get(0)); + Assert.assertEquals("rangerEndpoint", rangerEndpoint.getAllValues().get(0)); + Assert.assertEquals("hive", serviceName.getAllValues().get(0)); + RangerExportPolicyList actualPolicyList = rangerPolicyCapture.getAllValues().get(0); + Assert.assertEquals(rangerPolicyList.getMetaDataInfo(), actualPolicyList.getMetaDataInfo()); + //Deny policy is added + Assert.assertEquals(1, actualPolicyList.getListSize()); + } }