From 023a5cdf20a95200400a3d18156d50794c06e006 Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 18 Jun 2018 16:19:39 -0700 Subject: [PATCH] [HIVE-19941] Skip Adding virtual columns to masking query for non-native tables --- .../SQLStdHiveAuthorizationValidatorForTest.java | 4 ++ .../hadoop/hive/ql/parse/MaskAndFilterInfo.java | 4 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +- .../org/apache/hadoop/hive/ql/parse/TableMask.java | 2 +- .../queries/clientpositive/druidmini_masking.q | 22 +++++++ .../clientpositive/druid/druidmini_masking.q.out | 76 ++++++++++++++++++++++ 6 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/druidmini_masking.q create mode 100644 ql/src/test/results/clientpositive/druid/druidmini_masking.q.out diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java index b83d17cd7a..d883e4b17c 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java @@ -150,6 +150,10 @@ public boolean needTransform() { } else if (privObj.getObjectName().equals("masking_acid_no_masking")) { // testing acid usage when no masking/filtering is present needRewritePrivObjs.add(privObj); + } else if (privObj.getObjectName().equals("masking_test_druid")) { + // testing druid queries row filtering is present + privObj.setRowFilterExpression("key > 10"); + needRewritePrivObjs.add(privObj); } } return needRewritePrivObjs; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java index 40679dc862..5e252aaff7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java @@ -26,15 +26,17 @@ String alias; ASTNode astNode; boolean isView; + boolean isNonNative; public MaskAndFilterInfo(List colTypes, String additionalTabInfo, String alias, - ASTNode astNode, boolean isView) { + ASTNode astNode, boolean isView, boolean isNonNative) { super(); this.colTypes = colTypes; this.additionalTabInfo = additionalTabInfo; this.alias = alias; this.astNode = astNode; this.isView = isView; + this.isNonNative = isNonNative; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 284fcac265..5d18bd5298 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -11856,7 +11856,7 @@ private static void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set 0 && !ignoredTokens.contains(astNode.getToken().getType())) { for (Node child : astNode.getChildren()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java index 4ffff38765..ee93cf65fd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java @@ -124,7 +124,7 @@ public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFi sb.append("(SELECT *"); } - if (!maskAndFilterInfo.isView) { + if (!maskAndFilterInfo.isView && !maskAndFilterInfo.isNonNative) { // put all virtual columns in RowResolver. Iterator vcs = VirtualColumn.getRegistry(conf).iterator(); while (vcs.hasNext()) { diff --git a/ql/src/test/queries/clientpositive/druidmini_masking.q b/ql/src/test/queries/clientpositive/druidmini_masking.q new file mode 100644 index 0000000000..dbc8df5b1d --- /dev/null +++ b/ql/src/test/queries/clientpositive/druidmini_masking.q @@ -0,0 +1,22 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +CREATE TABLE masking_test_druid +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +AS + SELECT cast(current_timestamp() AS timestamp with local time zone) AS `__time`, + cast(username AS string) AS username, + cast(double1 AS double) AS double1, + cast(key AS int) AS key + FROM TABLE ( + VALUES + ('alfred', 10.30, -2), + ('bob', 3.14, null), + ('bonnie', null, 100), + ('calvin', null, null), + ('charlie', 15.8, 20)) as q (username, double1, key); + +explain select * from masking_test_druid; + +select * from masking_test_druid; diff --git a/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out b/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out new file mode 100644 index 0000000000..72647688b9 --- /dev/null +++ b/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out @@ -0,0 +1,76 @@ +PREHOOK: query: CREATE TABLE masking_test_druid +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +AS + SELECT cast(current_timestamp() AS timestamp with local time zone) AS `__time`, + cast(username AS string) AS username, + cast(double1 AS double) AS double1, + cast(key AS int) AS key + FROM TABLE ( + VALUES + ('alfred', 10.30, -2), + ('bob', 3.14, null), + ('bonnie', null, 100), + ('calvin', null, null), + ('charlie', 15.8, 20)) as q (username, double1, key) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test_druid +POSTHOOK: query: CREATE TABLE masking_test_druid +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +AS + SELECT cast(current_timestamp() AS timestamp with local time zone) AS `__time`, + cast(username AS string) AS username, + cast(double1 AS double) AS double1, + cast(key AS int) AS key + FROM TABLE ( + VALUES + ('alfred', 10.30, -2), + ('bob', 3.14, null), + ('bonnie', null, 100), + ('calvin', null, null), + ('charlie', 15.8, 20)) as q (username, double1, key) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test_druid +POSTHOOK: Lineage: masking_test_druid.__time SIMPLE [] +POSTHOOK: Lineage: masking_test_druid.double1 SCRIPT [] +POSTHOOK: Lineage: masking_test_druid.key SCRIPT [] +POSTHOOK: Lineage: masking_test_druid.username SCRIPT [] +PREHOOK: query: explain select * from masking_test_druid +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test_druid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: masking_test_druid + properties: + druid.fieldNames __time,username,double1,key + druid.fieldTypes timestamp with local time zone,string,double,int + druid.query.json {"queryType":"scan","dataSource":"default.masking_test_druid","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"bound","dimension":"key","lower":"10","lowerStrict":true,"ordering":"numeric"},"columns":["__time","username","double1","key"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: __time (type: timestamp with local time zone), username (type: string), double1 (type: double), key (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink + +PREHOOK: query: select * from masking_test_druid +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test_druid +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from masking_test_druid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test_druid +POSTHOOK: Output: hdfs://### HDFS PATH ### +2018-06-18 16:07:12.596 US/Pacific bonnie 0.0 100 +2018-06-18 16:07:12.596 US/Pacific charlie 15.800000190734863 20 -- 2.15.1 (Apple Git-101)