diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectList.java b/fe/src/main/java/org/apache/impala/analysis/SelectList.java index 3ac383035..9c9a2bed7 100644 --- a/fe/src/main/java/org/apache/impala/analysis/SelectList.java +++ b/fe/src/main/java/org/apache/impala/analysis/SelectList.java @@ -22,6 +22,8 @@ import java.util.List; import org.apache.impala.common.AnalysisException; import org.apache.impala.rewrite.ExprRewriter; +import org.apache.impala.rewrite.AddColumnMaskRule; +import org.apache.impala.rewrite.ExprRewriteRule; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -94,9 +96,10 @@ public class SelectList { public void rewriteExprs(ExprRewriter rewriter, Analyzer analyzer) throws AnalysisException { + ExprRewriteRule maskRule = AddColumnMaskRule.INSTANCE; for (SelectListItem item: items_) { if (item.isStar()) continue; - item.setExpr(rewriter.rewrite(item.getExpr(), analyzer)); + item.setExpr(rewriter.rewriteWithTempRule(item.getExpr(), analyzer, maskRule)); } } diff --git a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationChecker.java b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationChecker.java index 5cffa0213..7ebb49090 100644 --- a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationChecker.java +++ b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationChecker.java @@ -21,6 +21,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.hadoop.security.UserGroupInformation; import org.apache.impala.analysis.AnalysisContext.AnalysisResult; +import org.apache.impala.analysis.Parser; +import org.apache.impala.analysis.StatementBase; +import org.apache.impala.common.AnalysisException; import org.apache.impala.authorization.Authorizable; import org.apache.impala.authorization.Authorizable.Type; import org.apache.impala.authorization.AuthorizationChecker; @@ -37,11 +40,15 @@ import org.apache.impala.common.InternalException; import org.apache.impala.thrift.TSessionState; import org.apache.impala.util.EventSequence; import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.commons.lang3.StringUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -278,13 +285,54 @@ public class RangerAuthorizationChecker extends BaseAuthorizationChecker { .table(tableName) .column(columnName) .build(); +// RangerHiveAuthorizer.java:1105 RangerAccessRequest req = new RangerAccessRequestImpl(resource, SELECT_ACCESS_TYPE, user.getShortName(), getUserGroups(user)); - if (plugin_.evalDataMaskPolicies(req, null).isMaskEnabled()) { - throw new AuthorizationException(String.format( - "Impala does not support column masking yet. Column masking is enabled on " + - "column: %s.%s.%s", dbName, tableName, columnName)); + +// if (plugin_.evalDataMaskPolicies(req, null).isMaskEnabled()) { +// throw new AuthorizationException(String.format( +// "Impala does not support column masking yet. Column masking is enabled on " + +// "column: %s.%s.%s", dbName, tableName, columnName)); +// } + } + + public String getColumnMask(User user, String dbName, String tableName, + String columnName) { + try + { + RangerAccessResourceImpl resource = new RangerImpalaResourceBuilder() + .database(dbName) + .table(tableName) + .column(columnName) + .build(); + RangerAccessRequest req = new RangerAccessRequestImpl(resource, + SELECT_ACCESS_TYPE, user.getShortName(), getUserGroups(user)); + RangerAccessResult result = plugin_.evalDataMaskPolicies(req, null); + if(result == null || !result.isMaskEnabled()) + return null; + +// See org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizer.addCellValueTransformerAndCheckIfTransformed + String maskType = result.getMaskType(); + + if(StringUtils.equalsIgnoreCase(maskType, RangerPolicy.MASK_TYPE_NULL)) { + return "NULL"; + } else if(StringUtils.equalsIgnoreCase(maskType, RangerPolicy.MASK_TYPE_NONE)) { + return null; + } else if(StringUtils.equalsIgnoreCase(maskType, RangerPolicy.MASK_TYPE_CUSTOM)) { + return result.getMaskedValue().replace("{col}", columnName); +// Hive gets mask expression from ranger-servicedef-hive.json. +// We will harcode SQL here instead + } else if(StringUtils.equalsIgnoreCase(maskType, "MASK_SHOW_LAST_4")) { + return "RIGHT("+columnName+", 4)"; + } else { + RangerDataMaskTypeDef maskTypeDef = result.getMaskTypeDef(); + if (maskTypeDef != null) { + return maskTypeDef.getTransformer().replace("{col}", columnName); + } + } + } catch (InternalException e) { } + return null; } /** @@ -300,10 +348,19 @@ public class RangerAuthorizationChecker extends BaseAuthorizationChecker { .build(); RangerAccessRequest req = new RangerAccessRequestImpl(resource, SELECT_ACCESS_TYPE, user.getShortName(), getUserGroups(user)); + +/* TODO: Calls to retrieve row filter + RangerAccessResult result = plugin_.evalRowFilterPolicies(req, null); + if(result != null && result.isRowFilterEnabled()) + { + String rowFilterExpr = result.getFilterExpr(); + } +*/ + if (plugin_.evalRowFilterPolicies(req, null).isRowFilterEnabled()) { - throw new AuthorizationException(String.format( - "Impala does not support row filtering yet. Row filtering is enabled " + - "on table: %s.%s", dbName, tableName)); +// throw new AuthorizationException(String.format( +// "Impala does not support row filtering yet. Row filtering is enabled " + +// "on table: %s.%s", dbName, tableName)); } } diff --git a/fe/src/main/java/org/apache/impala/rewrite/AddColumnMaskRule.java b/fe/src/main/java/org/apache/impala/rewrite/AddColumnMaskRule.java new file mode 100644 index 000000000..928d44721 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/rewrite/AddColumnMaskRule.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.rewrite; + +import org.apache.impala.analysis.Analyzer; +import org.apache.impala.analysis.Path; +import org.apache.impala.analysis.SlotRef; +import org.apache.impala.analysis.SlotDescriptor; +import org.apache.impala.analysis.Expr; +//import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef; +import org.apache.impala.authorization.ranger.RangerAuthorizationFactory; +import org.apache.impala.authorization.ranger.RangerAuthorizationChecker; +import org.apache.impala.analysis.Parser; +import org.apache.impala.analysis.StatementBase; +import org.apache.impala.analysis.SelectStmt; +import org.apache.impala.common.AnalysisException; +import org.apache.impala.catalog.FeTable; + +/** + * Replace columns with mask values + * + * Examples: + * mask=reverse({col}) + * c1 -> reverse(c1) + */ +public class AddColumnMaskRule implements ExprRewriteRule { + public static ExprRewriteRule INSTANCE = new AddColumnMaskRule(); + + @Override + public Expr apply(Expr expr, Analyzer analyzer) { + if (!expr.isAnalyzed()) return expr; + + // TODO: add normalization for other expr types. + if (expr instanceof SlotRef) { + return addColumnMask((SlotRef) expr, analyzer); + } + return expr; + } + + private Expr addColumnMask(SlotRef expr, Analyzer analyzer) { + Expr retExpr = expr; + Path path = expr.getDesc().getPath(); + FeTable tab = path.getRootTable(); + // tab can be null for invalid reference case: + // WITH foo AS (SELECT c1 FROM t1) SELECT c1 FROM FOO + if(tab == null) + return retExpr; + String dbName = tab.getDb().getName(); + String tabName = tab.getName(); + String colName = path.destColumn().getName(); + String colType = expr.getType().toSql(); +// TODO: Add generic auth interfaces and overrides + RangerAuthorizationChecker authChecker = (RangerAuthorizationChecker)((RangerAuthorizationFactory)analyzer.getAuthzFactory()).newAuthorizationChecker(null); + String columnMask = authChecker.getColumnMask(analyzer.getUser(), dbName, tabName, colName); + if(columnMask != null) { + SelectStmt maskStmt; + try { + maskStmt = (SelectStmt)Parser.parse("SELECT CAST("+columnMask+" AS "+colType +")"); + retExpr = maskStmt.getSelectList().getItems().get(0).getExpr(); + } catch (AnalysisException e) { + } + } + return retExpr; + } +} diff --git a/fe/src/main/java/org/apache/impala/rewrite/ExprRewriter.java b/fe/src/main/java/org/apache/impala/rewrite/ExprRewriter.java index 37c092733..b721d5435 100644 --- a/fe/src/main/java/org/apache/impala/rewrite/ExprRewriter.java +++ b/fe/src/main/java/org/apache/impala/rewrite/ExprRewriter.java @@ -58,6 +58,20 @@ public class ExprRewriter { return rewrittenExpr; } + public Expr rewriteWithTempRule(Expr expr, Analyzer analyzer, ExprRewriteRule tempRule) throws AnalysisException { + // Keep applying the rule list until no rule has made any changes. + int oldNumChanges; + Expr rewrittenExpr = expr; + do { + oldNumChanges = numChanges_; + rewrittenExpr = applyRuleRepeatedly(rewrittenExpr, tempRule, analyzer); + for (ExprRewriteRule rule: rules_) { + rewrittenExpr = applyRuleRepeatedly(rewrittenExpr, rule, analyzer); + } + } while (oldNumChanges != numChanges_); + return rewrittenExpr; + } + /** * Applies 'rule' on the Expr tree rooted at 'expr' until there are no more changes. * Returns the transformed Expr or 'expr' if there were no changes.