diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java index 9f47f84..acf2663 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java @@ -373,7 +373,7 @@ private void checkSingleTableInput(List inputs) { verify(mockedAuthorizer).checkPrivileges(any(HiveOperationType.class), inputsCapturer.capture(), outputsCapturer.capture(), - any(HiveAuthzContext.class)); + any(QueryContext.class)); return new ImmutablePair(inputsCapturer.getValue(), outputsCapturer.getValue()); } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java index 5922a8c..0209044 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java @@ -77,7 +77,7 @@ protected abstract class AuthorizerWithFilterCmdImpl implements HiveAuthorizer { @Override public List filterListCmdObjects(List listObjs, - HiveAuthzContext context) throws HiveAuthzPluginException, HiveAccessControlException { + QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException { // capture arguments in static filterArguments = listObjs; // return static variable with results, if it is set to some set of @@ -101,7 +101,7 @@ public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreC try { Mockito.when( mockedAuthorizer.filterListCmdObjects((List) any(), - (HiveAuthzContext) any())).thenCallRealMethod(); + (QueryContext) any())).thenCallRealMethod(); } catch (Exception e) { org.junit.Assert.fail("Caught exception " + e); } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java index c43776b..ec74324 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java @@ -36,12 +36,12 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizerFactory; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactory; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -110,19 +110,16 @@ private void verifyContextContents(final String cmd, String ctxCmd) throws Excep stmt.close(); hs2Conn.close(); - ArgumentCaptor contextCapturer = ArgumentCaptor - .forClass(HiveAuthzContext.class); + ArgumentCaptor contextCapturer = ArgumentCaptor + .forClass(QueryContext.class); verify(mockedAuthorizer).checkPrivileges(any(HiveOperationType.class), Matchers.anyListOf(HivePrivilegeObject.class), Matchers.anyListOf(HivePrivilegeObject.class), contextCapturer.capture()); - HiveAuthzContext context = contextCapturer.getValue(); + QueryContext context = contextCapturer.getValue(); assertEquals("Command ", ctxCmd, context.getCommandString()); - assertTrue("ip address pattern check", context.getIpAddress().matches("[.:a-fA-F0-9]+")); - // ip address size check - check for something better than non zero - assertTrue("ip address size check", context.getIpAddress().length() > 7); } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcMetadataApiAuth.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcMetadataApiAuth.java index 692bfa0..f67f5c3 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcMetadataApiAuth.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestJdbcMetadataApiAuth.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizerFactory; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizerImpl; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactory; @@ -76,7 +76,7 @@ public TestAuthValidator(HiveMetastoreClientFactory metastoreClientFactory, Hive @Override public void checkPrivileges(HiveOperationType hiveOpType, List inputHObjs, - List outputHObjs, HiveAuthzContext context) + List outputHObjs, QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException { if (!allowActions) { throw new HiveAccessControlException(DENIED_ERR); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyAuthenticator.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyAuthenticator.java index a296ac5..8dc801f 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyAuthenticator.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyAuthenticator.java @@ -67,4 +67,9 @@ public void setSessionState(SessionState ss) { //no op } + @Override + public String getUserIpAddress() { + return null; + } + } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java index 322834e..40b0185 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java @@ -105,4 +105,9 @@ public void setSessionState(SessionState arg0) { //no-op } + @Override + public String getUserIpAddress() { + return null; + } + } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java index c0387e2..31e1b97 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd; +import java.util.ArrayList; import java.util.List; import java.util.HashSet; import java.util.Set; @@ -28,7 +29,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactory; @@ -92,7 +93,7 @@ public boolean apply(@Nullable HivePrivilegeObject hivePrivilegeObject) { @Override public void checkPrivileges(HiveOperationType hiveOpType, List inputHObjs, - List outputHObjs, HiveAuthzContext context) throws HiveAuthzPluginException, + List outputHObjs, QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException { switch (hiveOpType) { case DFS: @@ -105,32 +106,45 @@ public void checkPrivileges(HiveOperationType hiveOpType, List needTransform(List privObjs) { + List needTrans = new ArrayList<>(); + for (HivePrivilegeObject privObj : privObjs) { + if ("masking_test".equals(privObj.getObjectName()) + || "masking_test_subq".equals(privObj.getObjectName())) { + needTrans.add(true); + } else { + needTrans.add(false); + } + } + return needTrans; } - public String getCellValueTransformer(String database, String table, String columnName) + public void applyRowFilterAndColumnMasking(List privObjs) throws SemanticException { - if (table.equals("masking_test") && columnName.equals("value")) { - return "reverse(value)"; + for (HivePrivilegeObject privObj : privObjs) { + if (privObj.getObjectName().equals("masking_test")) { + privObj.setRowFilterExpression("key % 2 = 0 and key < 10"); + List cellValueTransformers = new ArrayList<>(); + for (String columnName : privObj.getColumns()) { + if (columnName.equals("value")) { + cellValueTransformers.add("reverse(value)"); + } else { + cellValueTransformers.add(columnName); + } + } + privObj.setCellValueTransformers(cellValueTransformers); + } else if (privObj.getObjectName().equals("masking_test_subq")) { + privObj + .setRowFilterExpression("key in (select key from src where src.key = masking_test_subq.key)"); + } } - return columnName; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 7276e31..65ed1db 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -104,7 +104,7 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.security.authorization.AuthorizationUtils; import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivObjectActionType; @@ -847,8 +847,7 @@ private static void doAuthorizationV2(SessionState ss, HiveOperation op, Set(); noscan = partialscan = false; - tableMask = new TableMask(this, conf); tabNameToTabObject = new HashMap<>(); } @@ -10361,6 +10362,7 @@ private void walkASTMarkTABREF(ASTNode ast, Set cteAlias) throws SemanticException { Queue queue = new LinkedList<>(); queue.add(ast); + Map basicInfos = new LinkedHashMap<>(); while (!queue.isEmpty()) { ASTNode astNode = (ASTNode) queue.poll(); if (astNode.getToken().getType() == HiveParser.TOK_TABREF) { @@ -10368,7 +10370,7 @@ private void walkASTMarkTABREF(ASTNode ast, Set cteAlias) StringBuffer additionalTabInfo = new StringBuffer(); for (int index = 1; index < astNode.getChildCount(); index++) { ASTNode ct = (ASTNode) astNode.getChild(index); - // TODO: support TOK_TABLEBUCKETSAMPLE, TOK_TABLESPLITSAMPLE, and + // We support TOK_TABLEBUCKETSAMPLE, TOK_TABLESPLITSAMPLE, and // TOK_TABLEPROPERTIES if (ct.getToken().getType() == HiveParser.TOK_TABLEBUCKETSAMPLE || ct.getToken().getType() == HiveParser.TOK_TABLESPLITSAMPLE @@ -10408,14 +10410,13 @@ private void walkASTMarkTABREF(ASTNode ast, Set cteAlias) throw new SemanticException("Table " + tabIdName + " is not found."); } - if (tableMask.needTransform(table.getDbName(), table.getTableName())) { - replacementText = tableMask.create(table, additionalTabInfo.toString(), alias); - } - if (replacementText != null) { - tableMask.setNeedsRewrite(true); - // we replace the tabref with replacementText here. - tableMask.addTableMasking(astNode, replacementText); + List columns = new ArrayList<>(); + for (FieldSchema col : table.getAllCols()) { + columns.add(col.getName()); } + + basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), columns), + new MaskAndFilterInfo(additionalTabInfo.toString(), alias, astNode)); } if (astNode.getChildCount() > 0 && !ignoredTokens.contains(astNode.getToken().getType())) { for (Node child : astNode.getChildren()) { @@ -10423,8 +10424,26 @@ private void walkASTMarkTABREF(ASTNode ast, Set cteAlias) } } } + List basicPrivObjs = new ArrayList<>(); + basicPrivObjs.addAll(basicInfos.keySet()); + List needTrans = tableMask.needTransform(basicPrivObjs); + List needRewriteObjs = new ArrayList<>(); + for (int index = 0; index < needTrans.size(); index++) { + if (needTrans.get(index)) { + needRewriteObjs.add(basicPrivObjs.get(index)); + } + } + if (needRewriteObjs.size() > 0) { + tableMask.setNeedsRewrite(true); + tableMask.applyRowFilterAndColumnMasking(needRewriteObjs); + for (HivePrivilegeObject privObj : needRewriteObjs) { + MaskAndFilterInfo info = basicInfos.get(privObj); + String replacementText = tableMask.create(privObj, info); + tableMask.addTableMasking(info.astNode, replacementText); + } + } } - + // We walk through the AST. // We replace all the TOK_TABREF by adding additional masking and filter if // the table needs to be masked or filtered. @@ -10544,6 +10563,7 @@ else if(ast.getChild(0).getType() == HiveParser.TOK_FALSE) { // masking and filtering should be done here // the basic idea is similar to unparseTranslator. + tableMask = new TableMask(this, conf); if (!unparseTranslator.isEnabled() && tableMask.isEnabled()) { child = rewriteASTWithMaskAndFilter(ast); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java index c47c2bd..516b6fe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.parse; +import java.util.ArrayList; import java.util.List; import org.antlr.runtime.TokenRewriteStream; @@ -24,6 +25,9 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +49,10 @@ public TableMask(SemanticAnalyzer analyzer, HiveConf conf) throws SemanticException { try { authorizer = SessionState.get().getAuthorizerV2(); - if (authorizer != null && needTransform()) { + String cmdString = analyzer.ctx.getCmd(); + QueryContext.Builder ctxBuilder = new QueryContext.Builder(); + ctxBuilder.setCommandString(cmdString); + if (authorizer != null && needTransform(ctxBuilder.build())) { enable = true; translator = new UnparseTranslator(conf); translator.enable(); @@ -56,64 +63,68 @@ public TableMask(SemanticAnalyzer analyzer, HiveConf conf) throws SemanticExcept } } - private String createRowMask(String db, String name) throws SemanticException { - return authorizer.getRowFilterExpression(db, name); - } - - private String createExpressions(String db, String tbl, String colName) throws SemanticException { - return authorizer.getCellValueTransformer(db, tbl, colName); + public void applyRowFilterAndColumnMasking(List privObjs) + throws SemanticException { + authorizer.applyRowFilterAndColumnMasking(privObjs); } public boolean isEnabled() throws SemanticException { return enable; } - public boolean needTransform() throws SemanticException { - return authorizer.needTransform(); + public boolean needTransform(QueryContext context) throws SemanticException { + return authorizer.needTransform(context); } - public boolean needTransform(String database, String table) throws SemanticException { - return authorizer.needTransform(database, table); + public List needTransform(List privObjects) + throws SemanticException { + return authorizer.needTransform(privObjects); } - public String create(Table table, String additionalTabInfo, String alias) throws SemanticException { - String db = table.getDbName(); - String tbl = table.getTableName(); + public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFilterInfo) + throws SemanticException { StringBuilder sb = new StringBuilder(); sb.append("(SELECT "); - List cols = table.getAllCols(); boolean firstOne = true; - for (FieldSchema fs : cols) { - if (!firstOne) { - sb.append(", "); - } else { - firstOne = false; + List exprs = privObject.getCellValueTransformers(); + if (exprs != null) { + for (int index = 0; index < exprs.size(); index++) { + String expr = exprs.get(index); + if (!firstOne) { + sb.append(", "); + } else { + firstOne = false; + } + sb.append(expr + " AS " + privObject.getColumns().get(index)); } - String colName = fs.getName(); - String expr = createExpressions(db, tbl, colName); - if (expr == null) { - sb.append(colName); - } else { - sb.append(expr + " AS " + colName); + } else { + for (int index = 0; index < privObject.getColumns().size(); index++) { + String expr = privObject.getColumns().get(index); + if (!firstOne) { + sb.append(", "); + } else { + firstOne = false; + } + sb.append(expr); } } - sb.append(" FROM " + tbl); - sb.append(" " + additionalTabInfo); - String filter = createRowMask(db, tbl); + sb.append(" FROM " + privObject.getObjectName()); + sb.append(" " + maskAndFilterInfo.additionalTabInfo); + String filter = privObject.getRowFilterExpression(); if (filter != null) { sb.append(" WHERE " + filter); } - sb.append(")" + alias); + sb.append(")" + maskAndFilterInfo.alias); LOG.debug("TableMask creates `" + sb.toString() + "`"); return sb.toString(); } void addTableMasking(ASTNode node, String replacementText) throws SemanticException { - translator.addTranslation(node, replacementText); + translator.addTranslation(node, replacementText); } void applyTableMasking(TokenRewriteStream tokenRewriteStream) throws SemanticException { - translator.applyTranslations(tokenRewriteStream); + translator.applyTranslations(tokenRewriteStream); } public boolean needsRewrite() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandUtil.java index d98b30c..7971dab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandUtil.java @@ -25,7 +25,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; @@ -80,9 +80,8 @@ static CommandProcessorResponse authorizeCommand(SessionState ss, HiveOperationT static void authorizeCommandThrowEx(SessionState ss, HiveOperationType type, List command) throws HiveAuthzPluginException, HiveAccessControlException { HivePrivilegeObject commandObj = HivePrivilegeObject.createHivePrivilegeObject(command); - HiveAuthzContext.Builder ctxBuilder = new HiveAuthzContext.Builder(); + QueryContext.Builder ctxBuilder = new QueryContext.Builder(); ctxBuilder.setCommandString(Joiner.on(' ').join(command)); - ctxBuilder.setUserIpAddress(ss.getUserIpAddress()); ss.getAuthorizerV2().checkPrivileges(type, Arrays.asList(commandObj), null, ctxBuilder.build()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/HadoopDefaultAuthenticator.java b/ql/src/java/org/apache/hadoop/hive/ql/security/HadoopDefaultAuthenticator.java index 18e4e00..8a036ac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/HadoopDefaultAuthenticator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/HadoopDefaultAuthenticator.java @@ -81,4 +81,9 @@ public void setSessionState(SessionState ss) { //no op } + @Override + public String getUserIpAddress() { + return null; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/HiveAuthenticationProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/security/HiveAuthenticationProvider.java index 7befff8..761352a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/HiveAuthenticationProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/HiveAuthenticationProvider.java @@ -32,6 +32,8 @@ public String getUserName(); + public String getUserIpAddress(); + public List getGroupNames(); public void destroy() throws HiveException; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/SessionStateConfigUserAuthenticator.java b/ql/src/java/org/apache/hadoop/hive/ql/security/SessionStateConfigUserAuthenticator.java index 8c7809e..87f4afa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/SessionStateConfigUserAuthenticator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/SessionStateConfigUserAuthenticator.java @@ -71,4 +71,9 @@ public void setSessionState(SessionState sessionState) { this.sessionState = sessionState; } + @Override + public String getUserIpAddress() { + return this.sessionState.getUserIpAddress(); + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/SessionStateUserAuthenticator.java b/ql/src/java/org/apache/hadoop/hive/ql/security/SessionStateUserAuthenticator.java index a77e93f..8f10914 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/SessionStateUserAuthenticator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/SessionStateUserAuthenticator.java @@ -65,4 +65,9 @@ public void setSessionState(SessionState sessionState) { this.sessionState = sessionState; } + @Override + public String getUserIpAddress() { + return this.sessionState.getUserIpAddress(); + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java index 6bad99b..a9ad015 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java @@ -73,8 +73,7 @@ public AuthorizationMetaStoreFilterHook(HiveConf conf) { private List getFilteredObjects(List listObjs) throws MetaException { SessionState ss = SessionState.get(); - HiveAuthzContext.Builder authzContextBuilder = new HiveAuthzContext.Builder(); - authzContextBuilder.setUserIpAddress(ss.getUserIpAddress()); + QueryContext.Builder authzContextBuilder = new QueryContext.Builder(); try { return ss.getAuthorizerV2().filterListCmdObjects(listObjs, authzContextBuilder.build()); } catch (HiveAuthzPluginException e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java index 1b366c2..45dce97 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java @@ -34,21 +34,18 @@ * see HiveAuthorizer.checkPrivileges */ void checkPrivileges(HiveOperationType hiveOpType, List inputHObjs, - List outputHObjs, HiveAuthzContext context) throws HiveAuthzPluginException, HiveAccessControlException; + List outputHObjs, QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException; /** * see HiveAuthorizer.filterListCmdObjects */ List filterListCmdObjects(List listObjs, - HiveAuthzContext context); + QueryContext context); - public String getRowFilterExpression(String database, String table) throws SemanticException; + public void applyRowFilterAndColumnMasking(List privObjs) throws SemanticException; - public String getCellValueTransformer(String database, String table, String columnName) - throws SemanticException; + public boolean needTransform(QueryContext context); - public boolean needTransform(); - - public boolean needTransform(String database, String table); + public List needTransform(List privObjs); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java index 6e2ef8d..743b92f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java @@ -161,7 +161,7 @@ void revokeRole(List hivePrincipals, List roles, boolean * @throws HiveAccessControlException */ void checkPrivileges(HiveOperationType hiveOpType, List inputsHObjs, - List outputHObjs, HiveAuthzContext context) + List outputHObjs, QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException; @@ -175,7 +175,7 @@ void checkPrivileges(HiveOperationType hiveOpType, List inp * @throws HiveAccessControlException */ List filterListCmdObjects(List listObjs, - HiveAuthzContext context) + QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException; @@ -242,30 +242,35 @@ void checkPrivileges(HiveOperationType hiveOpType, List inp * user, role or location. */ /** - * getRowFilterExpression is called once for each table in a query. It expects - * a valid filter condition to be returned. Null indicates no filtering is + * applyRowFilterAndColumnMasking is called once for each table in a query. + * (part 1) It expects a valid filter condition to be returned. Null indicates no filtering is * required. * * Example: table foo(c int) -> "c > 0 && c % 2 = 0" * - * @param database - * the name of the database in which the table lives - * @param table - * the name of the table in question - * @return + * (part 2) It expects a valid expression as used in a select clause. Null + * is not a valid option. If no transformation is needed simply return the + * column name. + * + * Example: column a -> "a" (no transform) + * + * Example: column a -> "reverse(a)" (call the reverse function on a) + * + * Example: column a -> "5" (replace column a with the constant 5) + * * @throws SemanticException */ - public String getRowFilterExpression(String database, String table) throws SemanticException; + public void applyRowFilterAndColumnMasking(List privObjs) throws SemanticException; /** - * needTransform() is called once per user in a query. If the function returns + * needTransform(QueryContext context) is called once per user in a query. If the function returns * true a call to needTransform(String database, String table) will happen. * Returning false short-circuits the generation of row/column transforms. * * @return * @throws SemanticException */ - public boolean needTransform(); + public boolean needTransform(QueryContext context); /** * needTransform(String database, String table) is called once per table in a @@ -280,28 +285,7 @@ void checkPrivileges(HiveOperationType hiveOpType, List inp * @return * @throws SemanticException */ - public boolean needTransform(String database, String table); + public List needTransform(List privObjs); - /** - * getCellValueTransformer is called once per column in each table accessed by - * the query. It expects a valid expression as used in a select clause. Null - * is not a valid option. If no transformation is needed simply return the - * column name. - * - * Example: column a -> "a" (no transform) - * - * Example: column a -> "reverse(a)" (call the reverse function on a) - * - * Example: column a -> "5" (replace column a with the constant 5) - * - * @param database - * @param table - * @param columnName - * @return - * @throws SemanticException - */ - public String getCellValueTransformer(String database, String table, String columnName) - throws SemanticException; - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerImpl.java index c73d667..a91b608 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerImpl.java @@ -82,7 +82,7 @@ public void revokeRole(List hivePrincipals, List roles, @Override public void checkPrivileges(HiveOperationType hiveOpType, List inputHObjs, - List outputHObjs, HiveAuthzContext context) + List outputHObjs, QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException { authValidator.checkPrivileges(hiveOpType, inputHObjs, outputHObjs, context); } @@ -90,7 +90,7 @@ public void checkPrivileges(HiveOperationType hiveOpType, List filterListCmdObjects(List listObjs, - HiveAuthzContext context) throws HiveAuthzPluginException, HiveAccessControlException { + QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException { return authValidator.filterListCmdObjects(listObjs, context); } @@ -138,24 +138,18 @@ public void applyAuthorizationConfigPolicy(HiveConf hiveConf) throws HiveAuthzPl } @Override - public String getRowFilterExpression(String database, String table) throws SemanticException { - return authValidator.getRowFilterExpression(table, table); + public boolean needTransform(QueryContext context) { + return authValidator.needTransform(context); } @Override - public String getCellValueTransformer(String database, String table, String columnName) - throws SemanticException { - return authValidator.getCellValueTransformer(database, table, columnName); + public void applyRowFilterAndColumnMasking(List privObjs) throws SemanticException { + authValidator.applyRowFilterAndColumnMasking(privObjs); } @Override - public boolean needTransform() { - return authValidator.needTransform(); - } - - @Override - public boolean needTransform(String database, String table) { - return authValidator.needTransform(database, table); + public List needTransform(List privObjs) { + return authValidator.needTransform(privObjs); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java deleted file mode 100644 index 195e341..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.security.authorization.plugin; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; -import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving; - -/** - * Provides context information in authorization check call that can be used for - * auditing and/or authorization. - * It is an immutable class. Builder inner class is used instantiate it. - */ -@LimitedPrivate(value = { "Apache Argus (incubating)" }) -@Evolving -public final class HiveAuthzContext { - - public static class Builder { - private String userIpAddress; - private String commandString; - - /** - * Get user's ip address. This is set only if the authorization - * api is invoked from a HiveServer2 instance in standalone mode. - * @return ip address - */ - public String getUserIpAddress() { - return userIpAddress; - } - public void setUserIpAddress(String userIpAddress) { - this.userIpAddress = userIpAddress; - } - public String getCommandString() { - return commandString; - } - public void setCommandString(String commandString) { - this.commandString = commandString; - } - public HiveAuthzContext build(){ - return new HiveAuthzContext(this); - } - - - } - - private final String userIpAddress; - private final String commandString; - - private HiveAuthzContext(Builder builder) { - this.userIpAddress = builder.userIpAddress; - this.commandString = builder.commandString; - - } - - public String getIpAddress() { - return userIpAddress; - } - - public String getCommandString() { - return commandString; - } - - @Override - public String toString() { - return "HiveAuthzContext [userIpAddress=" + userIpAddress + ", commandString=" + commandString - + "]"; - } - -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java index 0364627..0614c85 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java @@ -107,6 +107,8 @@ private int compare(Collection o1, Collection o2) { private final List partKeys; private final List columns; private final HivePrivObjectActionType actionType; + private List cellValueTransformers; + private String rowFilterExpression; public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName) { this(type, dbname, objectName, HivePrivObjectActionType.OTHER); @@ -139,6 +141,10 @@ public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String o this(type, dbname, objectName, partKeys, columns, HivePrivObjectActionType.OTHER, commandParams); } + public HivePrivilegeObject(String dbname, String objectName, List columns) { + this(null, dbname, objectName, null, columns, null); + } + public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName, List partKeys, List columns, HivePrivObjectActionType actionType, List commandParams) { @@ -242,4 +248,20 @@ public String toString() { private String getDbObjectName(String dbname2, String objectName2) { return (dbname == null ? "" : dbname + ".") + objectName; } + + public List getCellValueTransformers() { + return cellValueTransformers; + } + + public void setCellValueTransformers(List cellValueTransformers) { + this.cellValueTransformers = cellValueTransformers; + } + + public String getRowFilterExpression() { + return rowFilterExpression; + } + + public void setRowFilterExpression(String rowFilterExpression) { + this.rowFilterExpression = rowFilterExpression; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveV1Authorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveV1Authorizer.java index c8aa9db..612160a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveV1Authorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveV1Authorizer.java @@ -66,7 +66,7 @@ public VERSION getVersion() { @Override public void checkPrivileges(HiveOperationType hiveOpType, List inputsHObjs, - List outputHObjs, HiveAuthzContext context) + List outputHObjs, QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException { throw new UnsupportedOperationException("Should not be called for v1 authorizer"); } @@ -391,31 +391,23 @@ public void applyAuthorizationConfigPolicy(HiveConf hiveConf) { @Override public List filterListCmdObjects(List listObjs, - HiveAuthzContext context) throws HiveAuthzPluginException, HiveAccessControlException { + QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException { // do no filtering in old authorizer return listObjs; } - - @Override - public String getRowFilterExpression(String database, String table) throws SemanticException { - return null; - } - - @Override - public boolean needTransform() { + public boolean needTransform(QueryContext context) { return false; } @Override - public boolean needTransform(String database, String table) { - return false; + public void applyRowFilterAndColumnMasking(List privObjs) throws SemanticException { + } @Override - public String getCellValueTransformer(String database, String table, String columnName) - throws SemanticException { + public List needTransform(List privObjs) { return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/QueryContext.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/QueryContext.java new file mode 100644 index 0000000..318343c --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/QueryContext.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.security.authorization.plugin; + +import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; +import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving; + +/** + * Provides context information in authorization check call that can be used for + * auditing and/or authorization. + * It is an immutable class. Builder inner class is used instantiate it. + */ +@LimitedPrivate(value = { "Apache Argus (incubating)" }) +@Evolving +public final class QueryContext { + + public static class Builder { + private String commandString; + + public String getCommandString() { + return commandString; + } + public void setCommandString(String commandString) { + this.commandString = commandString; + } + public QueryContext build(){ + return new QueryContext(this); + } + } + + private final String commandString; + + private QueryContext(Builder builder) { + this.commandString = builder.commandString; + } + + public String getCommandString() { + return commandString; + } + + @Override + public String toString() { + return "QueryContext [commandString=" + commandString + "]"; + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/DummyHiveAuthorizationValidator.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/DummyHiveAuthorizationValidator.java index e4ddc9b..e4e6b3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/DummyHiveAuthorizationValidator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/DummyHiveAuthorizationValidator.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizationValidator; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; @@ -39,36 +39,31 @@ @Override public void checkPrivileges(HiveOperationType hiveOpType, List inputHObjs, - List outputHObjs, HiveAuthzContext context) + List outputHObjs, QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException { // no-op } @Override public List filterListCmdObjects(List listObjs, - HiveAuthzContext context) { + QueryContext context) { return listObjs; } - @Override - public String getRowFilterExpression(String database, String table) throws SemanticException { - return null; - } @Override - public String getCellValueTransformer(String database, String table, String columnName) - throws SemanticException { - return null; + public boolean needTransform(QueryContext context) { + return false; } @Override - public boolean needTransform() { - return false; + public void applyRowFilterAndColumnMasking(List privObjs) throws SemanticException { + } @Override - public boolean needTransform(String database, String table) { - return false; + public List needTransform(List privObjs) { + return null; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidator.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidator.java index c5d60b3..764b6ad 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidator.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizationValidator; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext.CLIENT_TYPE; @@ -65,7 +65,7 @@ public SQLStdHiveAuthorizationValidator(HiveMetastoreClientFactory metastoreClie @Override public void checkPrivileges(HiveOperationType hiveOpType, List inputHObjs, - List outputHObjs, HiveAuthzContext context) + List outputHObjs, QueryContext context) throws HiveAuthzPluginException, HiveAccessControlException { if (LOG.isDebugEnabled()) { @@ -141,7 +141,7 @@ private void checkPrivileges(HiveOperationType hiveOpType, List filterListCmdObjects(List listObjs, - HiveAuthzContext context) { + QueryContext context) { if (LOG.isDebugEnabled()) { String msg = "Obtained following objects in filterListCmdObjects " + listObjs + " for user " + authenticator.getUserName() + ". Context Info: " + context; @@ -151,24 +151,18 @@ private void checkPrivileges(HiveOperationType hiveOpType, List privObjs) throws SemanticException { + } @Override - public boolean needTransform(String database, String table) { - return false; + public List needTransform(List privObjs) { + return null; } } diff --git a/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java b/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java index 285b4f9..c4a7e69 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; @@ -134,8 +134,7 @@ protected void authorizeMetaGets(HiveOperationType opType, List inpObjs, String cmdString) throws HiveSQLException { SessionState ss = SessionState.get(); - HiveAuthzContext.Builder ctxBuilder = new HiveAuthzContext.Builder(); - ctxBuilder.setUserIpAddress(ss.getUserIpAddress()); + QueryContext.Builder ctxBuilder = new QueryContext.Builder(); ctxBuilder.setCommandString(cmdString); try { ss.getAuthorizerV2().checkPrivileges(opType, inpObjs, null,