From d7c299beb7021df220708f4ac7d47b70919d6744 Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 21 Jun 2018 12:04:41 -0700 Subject: [PATCH] [HIVE-19941] Skip Adding virtual columns to masking query for non-native tables --- .../test/resources/testconfiguration.properties | 3 +- .../SQLStdHiveAuthorizationValidatorForTest.java | 4 ++ .../hadoop/hive/ql/parse/MaskAndFilterInfo.java | 4 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +- .../org/apache/hadoop/hive/ql/parse/TableMask.java | 2 +- .../queries/clientpositive/druidmini_masking.q | 22 +++++++ .../clientpositive/druid/druidmini_masking.q.out | 76 ++++++++++++++++++++++ 7 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/druidmini_masking.q create mode 100644 ql/src/test/results/clientpositive/druid/druidmini_masking.q.out diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 93e2a445f4..3745f0cb02 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -1687,7 +1687,8 @@ druid.query.files=druidmini_test1.q,\ druidmini_expressions.q,\ druidmini_extractTime.q,\ druidmini_test_alter.q,\ - druidmini_floorTime.q + druidmini_floorTime.q, \ + druidmini_masking.q druid.kafka.query.files=druidkafkamini_basic.q diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java index b83d17cd7a..d883e4b17c 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java @@ -150,6 +150,10 @@ public boolean needTransform() { } else if (privObj.getObjectName().equals("masking_acid_no_masking")) { // testing acid usage when no masking/filtering is present needRewritePrivObjs.add(privObj); + } else if (privObj.getObjectName().equals("masking_test_druid")) { + // testing druid queries row filtering is present + privObj.setRowFilterExpression("key > 10"); + needRewritePrivObjs.add(privObj); } } return needRewritePrivObjs; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java index 40679dc862..5e252aaff7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java @@ -26,15 +26,17 @@ String alias; ASTNode astNode; boolean isView; + boolean isNonNative; public MaskAndFilterInfo(List colTypes, String additionalTabInfo, String alias, - ASTNode astNode, boolean isView) { + ASTNode astNode, boolean isView, boolean isNonNative) { super(); this.colTypes = colTypes; this.additionalTabInfo = additionalTabInfo; this.alias = alias; this.astNode = astNode; this.isView = isView; + this.isNonNative = isNonNative; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 779ca7dfca..37e1a73b51 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -11862,7 +11862,7 @@ private static void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set 0 && !ignoredTokens.contains(astNode.getToken().getType())) { for (Node child : astNode.getChildren()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java index 4ffff38765..ee93cf65fd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java @@ -124,7 +124,7 @@ public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFi sb.append("(SELECT *"); } - if (!maskAndFilterInfo.isView) { + if (!maskAndFilterInfo.isView && !maskAndFilterInfo.isNonNative) { // put all virtual columns in RowResolver. Iterator vcs = VirtualColumn.getRegistry(conf).iterator(); while (vcs.hasNext()) { diff --git a/ql/src/test/queries/clientpositive/druidmini_masking.q b/ql/src/test/queries/clientpositive/druidmini_masking.q new file mode 100644 index 0000000000..f0f2c0cbf6 --- /dev/null +++ b/ql/src/test/queries/clientpositive/druidmini_masking.q @@ -0,0 +1,22 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +CREATE TABLE masking_test_druid +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +AS + SELECT cast(current_timestamp() AS timestamp with local time zone) AS `__time`, + cast(username AS string) AS username, + cast(double1 AS double) AS double1, + cast(key AS int) AS key + FROM TABLE ( + VALUES + ('alfred', 10.30, -2), + ('bob', 3.14, null), + ('bonnie', null, 100), + ('calvin', null, null), + ('charlie', 15.8, 20)) as q (username, double1, key); + +explain select username, key from masking_test_druid; + +select username, key from masking_test_druid; diff --git a/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out b/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out new file mode 100644 index 0000000000..1aad9677a0 --- /dev/null +++ b/ql/src/test/results/clientpositive/druid/druidmini_masking.q.out @@ -0,0 +1,76 @@ +PREHOOK: query: CREATE TABLE masking_test_druid +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +AS + SELECT cast(current_timestamp() AS timestamp with local time zone) AS `__time`, + cast(username AS string) AS username, + cast(double1 AS double) AS double1, + cast(key AS int) AS key + FROM TABLE ( + VALUES + ('alfred', 10.30, -2), + ('bob', 3.14, null), + ('bonnie', null, 100), + ('calvin', null, null), + ('charlie', 15.8, 20)) as q (username, double1, key) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test_druid +POSTHOOK: query: CREATE TABLE masking_test_druid +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +AS + SELECT cast(current_timestamp() AS timestamp with local time zone) AS `__time`, + cast(username AS string) AS username, + cast(double1 AS double) AS double1, + cast(key AS int) AS key + FROM TABLE ( + VALUES + ('alfred', 10.30, -2), + ('bob', 3.14, null), + ('bonnie', null, 100), + ('calvin', null, null), + ('charlie', 15.8, 20)) as q (username, double1, key) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test_druid +POSTHOOK: Lineage: masking_test_druid.__time SIMPLE [] +POSTHOOK: Lineage: masking_test_druid.double1 SCRIPT [] +POSTHOOK: Lineage: masking_test_druid.key SCRIPT [] +POSTHOOK: Lineage: masking_test_druid.username SCRIPT [] +PREHOOK: query: explain select username, key from masking_test_druid +PREHOOK: type: QUERY +POSTHOOK: query: explain select username, key from masking_test_druid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: masking_test_druid + properties: + druid.fieldNames username,key + druid.fieldTypes string,int + druid.query.json {"queryType":"scan","dataSource":"default.masking_test_druid","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"bound","dimension":"key","lower":"10","lowerStrict":true,"ordering":"numeric"},"columns":["username","key"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: username (type: string), key (type: int) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: select username, key from masking_test_druid +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test_druid +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select username, key from masking_test_druid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test_druid +POSTHOOK: Output: hdfs://### HDFS PATH ### +bonnie 100 +charlie 20 -- 2.15.1 (Apple Git-101)