diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java new file mode 100644 index 0000000..06b3fcd --- /dev/null +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.security.authorization.plugin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.CommandNeedRetryException; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; +import org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; +import org.mockito.Mockito; + +/** + * Test HiveAuthorizer api invocation + */ +public class TestHiveAuthorizerCheckInvocation { + protected static HiveConf conf; + protected static Driver driver; + private static final String tableName = TestHiveAuthorizerCheckInvocation.class.getSimpleName(); + static HiveAuthorizer mockedAuthorizer; + + /** + * This factory creates a mocked HiveAuthorizer class. Use the mocked class to + * capture the argument passed to it in the test case. + */ + static class MockedHiveAuthorizerFactory implements HiveAuthorizerFactory { + @Override + public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreClientFactory, + HiveConf conf, HiveAuthenticationProvider authenticator) { + TestHiveAuthorizerCheckInvocation.mockedAuthorizer = Mockito.mock(HiveAuthorizer.class); + return TestHiveAuthorizerCheckInvocation.mockedAuthorizer; + } + + } + + @BeforeClass + public static void beforeTest() throws Exception { + conf = new HiveConf(); + + // Turn on mocked authorization + conf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, MockedHiveAuthorizerFactory.class.getName()); + conf.setVar(ConfVars.HIVE_AUTHENTICATOR_MANAGER, SessionStateUserAuthenticator.class.getName()); + conf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true); + conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + conf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + + SessionState.start(conf); + driver = new Driver(conf); + CommandProcessorResponse resp = driver.run("create table " + tableName + " (i int, j int, k string)"); + assertEquals(0, resp.getResponseCode()); + } + + @AfterClass + public static void afterTests() throws Exception { + driver.close(); + } + + @Test + public void testInputSomeColumnsUsed() throws HiveAuthzPluginException, HiveAccessControlException, + CommandNeedRetryException { + + reset(mockedAuthorizer); + int status = driver.compile("select i, k from " + tableName); + assertEquals(0, status); + + List inputs = getHivePrivilegeObjectInputs(); + checkSingleTableInput(inputs); + HivePrivilegeObject tableObj = inputs.get(0); + assertEquals("no of columns used", 2, tableObj.getColumns().size()); + Collections.sort(tableObj.getColumns()); + assertEquals("Columns used", Arrays.asList("i", "k"), tableObj.getColumns()); + } + + @Test + public void testInputAllColumnsUsed() throws HiveAuthzPluginException, HiveAccessControlException, + CommandNeedRetryException { + + reset(mockedAuthorizer); + int status = driver.compile("select * from " + tableName + " order by i"); + assertEquals(0, status); + + List inputs = getHivePrivilegeObjectInputs(); + checkSingleTableInput(inputs); + HivePrivilegeObject tableObj = inputs.get(0); + assertEquals("no of columns used", 3, tableObj.getColumns().size()); + + Collections.sort(tableObj.getColumns()); + assertEquals("Columns used", Arrays.asList("i", "j", "k"), tableObj.getColumns()); + } + + @Test + public void testInputNoColumnsUsed() throws HiveAuthzPluginException, HiveAccessControlException, + CommandNeedRetryException { + + reset(mockedAuthorizer); + int status = driver.compile("describe " + tableName); + assertEquals(0, status); + + List inputs = getHivePrivilegeObjectInputs(); + checkSingleTableInput(inputs); + HivePrivilegeObject tableObj = inputs.get(0); + assertNull("columns used", tableObj.getColumns()); + } + + private void checkSingleTableInput(List inputs) { + assertEquals("number of inputs", 1, inputs.size()); + + HivePrivilegeObject tableObj = inputs.get(0); + assertEquals("input type", HivePrivilegeObjectType.TABLE_OR_VIEW, tableObj.getType()); + assertTrue("table name", tableName.equalsIgnoreCase(tableObj.getObjectName())); + } + + /** + * @return the inputs passed in current call to authorizer.checkPrivileges + * @throws HiveAuthzPluginException + * @throws HiveAccessControlException + */ + private List getHivePrivilegeObjectInputs() throws HiveAuthzPluginException, + HiveAccessControlException { + // Create argument capturer + // a class variable cast to this generic of generic class + Class> class_listPrivObjects = (Class) List.class; + ArgumentCaptor> inputsCapturer = ArgumentCaptor + .forClass(class_listPrivObjects); + + verify(mockedAuthorizer).checkPrivileges(any(HiveOperationType.class), + inputsCapturer.capture(), Matchers.anyListOf(HivePrivilegeObject.class), + any(HiveAuthzContext.class)); + + return inputsCapturer.getValue(); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index e512199..6fb956d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -28,10 +28,12 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Queue; import java.util.Set; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FSDataInputStream; @@ -503,7 +505,14 @@ public static void doAuthorization(BaseSemanticAnalyzer sem, String command) Hive db = sem.getDb(); if (ss.isAuthorizationModeV2()) { - doAuthorizationV2(ss, op, inputs, outputs, command); + // get mapping of tables to columns used + Map> tab2Cols = new HashMap>(); + Map> part2Cols = new HashMap>(); + // partition level privileges are not checked, so leave following map empty + Map tableUsePartLevelAuth = new HashMap(); + getTablePartitionUsedColumns(op, sem, tab2Cols, part2Cols, tableUsePartLevelAuth); + + doAuthorizationV2(ss, op, inputs, outputs, command, tab2Cols); return; } if (op == null) { @@ -583,56 +592,9 @@ public static void doAuthorization(BaseSemanticAnalyzer sem, String command) } } - //for a select or create-as-select query, populate the partition to column (par2Cols) or - // table to columns mapping (tab2Cols) - if (op.equals(HiveOperation.CREATETABLE_AS_SELECT) - || op.equals(HiveOperation.QUERY)) { - SemanticAnalyzer querySem = (SemanticAnalyzer) sem; - ParseContext parseCtx = querySem.getParseContext(); - Map tsoTopMap = parseCtx.getTopToTable(); - - for (Map.Entry> topOpMap : querySem - .getParseContext().getTopOps().entrySet()) { - Operator topOp = topOpMap.getValue(); - if (topOp instanceof TableScanOperator - && tsoTopMap.containsKey(topOp)) { - TableScanOperator tableScanOp = (TableScanOperator) topOp; - Table tbl = tsoTopMap.get(tableScanOp); - List neededColumnIds = tableScanOp.getNeededColumnIDs(); - List columns = tbl.getCols(); - List cols = new ArrayList(); - for (int i = 0; i < neededColumnIds.size(); i++) { - cols.add(columns.get(neededColumnIds.get(i)).getName()); - } - //map may not contain all sources, since input list may have been optimized out - //or non-existent tho such sources may still be referenced by the TableScanOperator - //if it's null then the partition probably doesn't exist so let's use table permission - if (tbl.isPartitioned() && - tableUsePartLevelAuth.get(tbl.getTableName()) == Boolean.TRUE) { - String alias_id = topOpMap.getKey(); - - PrunedPartitionList partsList = PartitionPruner.prune(tableScanOp, - parseCtx, alias_id); - Set parts = partsList.getPartitions(); - for (Partition part : parts) { - List existingCols = part2Cols.get(part); - if (existingCols == null) { - existingCols = new ArrayList(); - } - existingCols.addAll(cols); - part2Cols.put(part, existingCols); - } - } else { - List existingCols = tab2Cols.get(tbl); - if (existingCols == null) { - existingCols = new ArrayList(); - } - existingCols.addAll(cols); - tab2Cols.put(tbl, existingCols); - } - } - } - } + getTablePartitionUsedColumns(op, sem, tab2Cols, part2Cols, tableUsePartLevelAuth); + + // cache the results for table authorization Set tableAuthChecked = new HashSet(); @@ -683,8 +645,65 @@ public static void doAuthorization(BaseSemanticAnalyzer sem, String command) } } + private static void getTablePartitionUsedColumns(HiveOperation op, BaseSemanticAnalyzer sem, + Map> tab2Cols, Map> part2Cols, + Map tableUsePartLevelAuth) throws HiveException { + // for a select or create-as-select query, populate the partition to column + // (par2Cols) or + // table to columns mapping (tab2Cols) + if (op.equals(HiveOperation.CREATETABLE_AS_SELECT) + || op.equals(HiveOperation.QUERY)) { + SemanticAnalyzer querySem = (SemanticAnalyzer) sem; + ParseContext parseCtx = querySem.getParseContext(); + Map tsoTopMap = parseCtx.getTopToTable(); + + for (Map.Entry> topOpMap : querySem + .getParseContext().getTopOps().entrySet()) { + Operator topOp = topOpMap.getValue(); + if (topOp instanceof TableScanOperator + && tsoTopMap.containsKey(topOp)) { + TableScanOperator tableScanOp = (TableScanOperator) topOp; + Table tbl = tsoTopMap.get(tableScanOp); + List neededColumnIds = tableScanOp.getNeededColumnIDs(); + List columns = tbl.getCols(); + List cols = new ArrayList(); + for (int i = 0; i < neededColumnIds.size(); i++) { + cols.add(columns.get(neededColumnIds.get(i)).getName()); + } + //map may not contain all sources, since input list may have been optimized out + //or non-existent tho such sources may still be referenced by the TableScanOperator + //if it's null then the partition probably doesn't exist so let's use table permission + if (tbl.isPartitioned() && + tableUsePartLevelAuth.get(tbl.getTableName()) == Boolean.TRUE) { + String alias_id = topOpMap.getKey(); + + PrunedPartitionList partsList = PartitionPruner.prune(tableScanOp, + parseCtx, alias_id); + Set parts = partsList.getPartitions(); + for (Partition part : parts) { + List existingCols = part2Cols.get(part); + if (existingCols == null) { + existingCols = new ArrayList(); + } + existingCols.addAll(cols); + part2Cols.put(part, existingCols); + } + } else { + List existingCols = tab2Cols.get(tbl); + if (existingCols == null) { + existingCols = new ArrayList(); + } + existingCols.addAll(cols); + tab2Cols.put(tbl, existingCols); + } + } + } + } + + } + private static void doAuthorizationV2(SessionState ss, HiveOperation op, HashSet inputs, - HashSet outputs, String command) throws HiveException { + HashSet outputs, String command, Map> tab2Cols) throws HiveException { HiveAuthzContext.Builder authzContextBuilder = new HiveAuthzContext.Builder(); @@ -696,11 +715,39 @@ private static void doAuthorizationV2(SessionState ss, HiveOperation op, HashSet HiveOperationType hiveOpType = getHiveOperationType(op); List inputsHObjs = getHivePrivObjects(inputs); + updateInputColumnInfo(inputsHObjs, tab2Cols); + List outputHObjs = getHivePrivObjects(outputs); ss.getAuthorizerV2().checkPrivileges(hiveOpType, inputsHObjs, outputHObjs, authzContextBuilder.build()); return; } + /** + * Add column information for input table objects + * @param inputsHObjs input HivePrivilegeObject + * @param tab2Cols table to used input columns mapping + */ + private static void updateInputColumnInfo(List inputsHObjs, + Map> tab2Cols) { + //create db+table name to used columns mapping + Map, List> tableName2Cols = + new HashMap, List>(); + + for (Entry> tab2Col : tab2Cols.entrySet()) { + Table tab = tab2Col.getKey(); + tableName2Cols.put(Pair.of(tab.getDbName(), tab.getTableName()), tab2Col.getValue()); + } + + for(HivePrivilegeObject inputObj : inputsHObjs){ + if(inputObj.getType() != HivePrivilegeObjectType.TABLE_OR_VIEW){ + // input columns are relevant only for tables or views + continue; + } + List cols = tableName2Cols.get(Pair.of(inputObj.getDbname(), inputObj.getObjectName())); + inputObj.setColumns(cols); + } + } + private static List getHivePrivObjects(HashSet privObjects) { List hivePrivobjs = new ArrayList(); if(privObjects == null){ diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java index 5733703..09a9127 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java @@ -77,7 +77,7 @@ private int compare(List o1, List o2) { private final String objectName; private final List commandParams; private final List partKeys; - private final List columns; + private List columns; private final HivePrivObjectActionType actionType; public HivePrivilegeObject(HivePrivilegeObjectType type, String dbname, String objectName) { @@ -200,4 +200,8 @@ private String getDbObjectName(String dbname2, String objectName2) { return (dbname == null ? "" : dbname + ".") + objectName; } + public void setColumns(List columnms) { + this.columns = columnms; + } + }