diff --git ql/src/java/org/apache/hadoop/hive/ql/Context.java ql/src/java/org/apache/hadoop/hive/ql/Context.java index fae2a12..d1d2789 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -125,6 +125,10 @@ private Heartbeater heartbeater; private boolean skipTableMasking; + + // Identify whether the query involves an UPDATE, DELETE or MERGE + private boolean isUpdateDeleteMerge; + /** * This determines the prefix of the * {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.Phase1Ctx#dest} @@ -955,4 +959,12 @@ public void resetOpContext(){ opContext = new CompilationOpContext(); sequencer = new AtomicInteger(); } + + public boolean getIsUpdateDeleteMerge() { + return isUpdateDeleteMerge; + } + + public void setIsUpdateDeleteMerge(boolean isUpdate) { + this.isUpdateDeleteMerge = isUpdate; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 6013218..2ffc130 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -398,6 +398,9 @@ "time."), DISTINCT_NOT_SUPPORTED(10285, "Distinct keyword is not support in current context"), NONACID_COMPACTION_NOT_SUPPORTED(10286, "Compaction is not allowed on non-ACID table {0}.{1}", true), + MASKING_FILTERING_ON_ACID_NOT_SUPPORTED(10287, + "Detected {0}.{1} has row masking/column filtering enabled, " + + "which is not supported for query involving ACID operations", true), UPDATEDELETE_PARSE_ERROR(10290, "Encountered parse error while parsing rewritten merge/update or " + "delete query"), diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index f053093..9c37af8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -10820,6 +10820,11 @@ private void walkASTMarkTABREF(ASTNode ast, Set cteAlias) .applyRowFilterAndColumnMasking(basicPrivObjs); if (needRewritePrivObjs != null && !needRewritePrivObjs.isEmpty()) { for (HivePrivilegeObject privObj : needRewritePrivObjs) { + // We don't support masking/filtering against ACID query at the moment + if (ctx.getIsUpdateDeleteMerge()) { + throw new SemanticException(ErrorMsg.MASKING_FILTERING_ON_ACID_NOT_SUPPORTED, + privObj.getDbname(), privObj.getObjectName()); + } MaskAndFilterInfo info = basicInfos.get(privObj); String replacementText = tableMask.create(privObj, info); if (replacementText != null) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java index 725f2ce..865c03a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -293,6 +293,7 @@ private ReparseResult parseRewrittenQuery(StringBuilder rewrittenQueryStr, Strin HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); rewrittenCtx = new Context(conf); rewrittenCtx.setExplainConfig(ctx.getExplainConfig()); + rewrittenCtx.setIsUpdateDeleteMerge(true); } catch (IOException e) { throw new SemanticException(ErrorMsg.UPDATEDELETE_IO_ERROR.getMsg()); } diff --git ql/src/test/queries/clientnegative/masking_acid_delete.q ql/src/test/queries/clientnegative/masking_acid_delete.q new file mode 100644 index 0000000..2785d7a --- /dev/null +++ ql/src/test/queries/clientnegative/masking_acid_delete.q @@ -0,0 +1,10 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +create table masking_test (key int, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true"); + +delete from masking_test where value='ddd'; diff --git ql/src/test/queries/clientnegative/masking_acid_merge.q ql/src/test/queries/clientnegative/masking_acid_merge.q new file mode 100644 index 0000000..50471e0 --- /dev/null +++ ql/src/test/queries/clientnegative/masking_acid_merge.q @@ -0,0 +1,15 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +create table nonacid (key int, value string) stored as orc; + +create table masking_test (key int, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true"); + +MERGE INTO masking_test as t using nonacid as s ON t.key = s.key +WHEN MATCHED AND s.key < 5 THEN DELETE +WHEN MATCHED AND s.key < 3 THEN UPDATE set key = 1 +WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.value); diff --git ql/src/test/queries/clientnegative/masking_acid_update.q ql/src/test/queries/clientnegative/masking_acid_update.q new file mode 100644 index 0000000..586d20b --- /dev/null +++ ql/src/test/queries/clientnegative/masking_acid_update.q @@ -0,0 +1,10 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +create table masking_test (key int, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true"); + +update masking_test set key=1 where value='ddd'; diff --git ql/src/test/results/clientnegative/masking_acid_delete.q.out ql/src/test/results/clientnegative/masking_acid_delete.q.out new file mode 100644 index 0000000..021f2cd --- /dev/null +++ ql/src/test/results/clientnegative/masking_acid_delete.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: create table masking_test (key int, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create table masking_test (key int, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +FAILED: SemanticException [Error 10287]: Detected default.masking_test has row masking/column filtering enabled, which is not supported for query involving ACID operations diff --git ql/src/test/results/clientnegative/masking_acid_merge.q.out ql/src/test/results/clientnegative/masking_acid_merge.q.out new file mode 100644 index 0000000..917c44a --- /dev/null +++ ql/src/test/results/clientnegative/masking_acid_merge.q.out @@ -0,0 +1,21 @@ +PREHOOK: query: create table nonacid (key int, value string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nonacid +POSTHOOK: query: create table nonacid (key int, value string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nonacid +PREHOOK: query: create table masking_test (key int, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create table masking_test (key int, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +FAILED: SemanticException [Error 10287]: Detected default.masking_test has row masking/column filtering enabled, which is not supported for query involving ACID operations diff --git ql/src/test/results/clientnegative/masking_acid_update.q.out ql/src/test/results/clientnegative/masking_acid_update.q.out new file mode 100644 index 0000000..021f2cd --- /dev/null +++ ql/src/test/results/clientnegative/masking_acid_update.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: create table masking_test (key int, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create table masking_test (key int, value string) +clustered by (value) into 2 buckets stored as orc +tblproperties ("transactional"="true") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +FAILED: SemanticException [Error 10287]: Detected default.masking_test has row masking/column filtering enabled, which is not supported for query involving ACID operations