commit 56bae9a783ff38f42217cb0c603ad4aabac43093 Author: Daniel Dai Date: Fri Oct 12 16:00:06 2018 -0700 HIVE-20731: keystore file in JdbcStorageHandler should be authorized diff --git a/data/files/test.jceks b/data/files/test.jceks new file mode 100644 index 0000000000000000000000000000000000000000..bfeefe158af622c16425cd09af7404bb0cb00631 GIT binary patch literal 988 zcmX?i?%X*B1_mZ5W@eBqNi8mkPf0CGs*F!8Ey*y_D@ZIZF3&GYVPIfPtGaw>?wYp@ zti?qP#>x4)dc~!AddWqV1ts}<1x5K~nLvGd!KsNksVV+RS*gh-Zuv#tsg(gm`6WP3 ze$mUl+=`f#G=I9 z;t~d?Xs2if0iYnr-qaN5{JfIXyplqOItC^m2F?PoD#x7k5(Xh3kOlfViFxVz!6ikR zdFj?Z4D7`~=K-Zb>X|DFN{c||t@$kegN-BP0mxkiz@Xqa;AP>+?%cll(|xX7CPtA` zkZM+-L$+;)#~B~G&eXWei29y*wSLvfQ;RGs zzjsKfvbP0l)t}z6fr+O?LLxq4=d~G$S8{7^$>qOu-9KTT=9<+dQ?H*_<}YCo3vhA` z&n(ICbuo3!O9?5;EXYZ9aSg^P0dzfP>>8rQ?t-$B61(+kZuk9~GHX#{H`dIq_CAN> z*^8B56TXIITfFjLyh5nTt>DT0W1(_2HaAUUcF8j;ce=Qy-dQ5{^2DQ-HT-;nA$8wb zF7vZ%FS>Si)%Eh9vzl!*qWkW>Gx0kr)jj*OReg?QOGi({Tla46V<+n?lQgHL@y`GB zdc~~Z^eCf)2X3^!?l+M;`7XM(^RHa3b;GLn6?ru{BN3+rue10TX(fXjSABW;UM8ek Jgl{YN0|4atb>aX3 literal 0 HcmV?d00001 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 1df5c74..c9df668 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -98,6 +99,9 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -2275,4 +2279,27 @@ public DDLDescWithWriteId getAcidDdlDesc() { public WriteEntity getAcidAnalyzeTable() { return null; } + + public void addPropertyReadEntry(Map tblProps, Set inputs) throws SemanticException { + if (tblProps.containsKey(Constants.JDBC_KEYSTORE)) { + try { + String keystore = tblProps.get(Constants.JDBC_KEYSTORE); + Configuration conf = new Configuration(); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, keystore); + boolean found = false; + for (CredentialProvider provider : CredentialProviderFactory.getProviders(conf)) + if (provider instanceof AbstractJavaKeyStoreProvider) { + Path path = ((AbstractJavaKeyStoreProvider) provider).getPath(); + inputs.add(toReadEntity(path)); + found = true; + } + if (!found) { + throw new SemanticException("Cannot recognize keystore " + keystore + ", only JavaKeyStoreProvider is " + + "supported"); + } + } catch (IOException e) { + throw new SemanticException(e); + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 29f6ecf..bba7d6c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1771,6 +1771,7 @@ else if(entry.getKey().equals("external") && entry.getValue().equals("true")){ alterTblDesc.setDropIfExists(true); } } else { + addPropertyReadEntry(mapProp, inputs); alterTblDesc = new AlterTableDesc(AlterTableTypes.ADDPROPS, partSpec, expectView); } alterTblDesc.setProps(mapProp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 31bc38e..b760049 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -13325,6 +13325,7 @@ ASTNode analyzeCreateTable( break; case HiveParser.TOK_TABLEPROPERTIES: tblProps = DDLSemanticAnalyzer.getProps((ASTNode) child.getChild(0)); + addPropertyReadEntry(tblProps, inputs); break; case HiveParser.TOK_TABLESERIALIZER: child = (ASTNode) child.getChild(0); diff --git a/ql/src/test/queries/clientnegative/authorization_jdbc_keystore.q b/ql/src/test/queries/clientnegative/authorization_jdbc_keystore.q new file mode 100644 index 0000000..63288f7 --- /dev/null +++ b/ql/src/test/queries/clientnegative/authorization_jdbc_keystore.q @@ -0,0 +1,28 @@ +--! qt:dataset: + +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; + +dfs -cp ${system:test.tmp.dir}/../../../../data/files/test.jceks ${system:test.tmp.dir}/test.jceks; +dfs -chmod 555 ${system:test.tmp.dir}/test.jceks; + +CREATE EXTERNAL TABLE ext_auth1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", + "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1;collation=TERRITORY_BASED:PRIMARY", + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password.keystore" = "jceks://file/${system:test.tmp.dir}/test.jceks", + "hive.sql.dbcp.password.key" = "test_derby_auth1.password", + "hive.sql.table" = "SIMPLE_DERBY_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +); diff --git a/ql/src/test/queries/clientpositive/external_jdbc_auth.q b/ql/src/test/queries/clientpositive/external_jdbc_auth.q index acfb298..f4cbe94 100644 --- a/ql/src/test/queries/clientpositive/external_jdbc_auth.q +++ b/ql/src/test/queries/clientpositive/external_jdbc_auth.q @@ -9,6 +9,18 @@ SELECT dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1;create=true','user1','passwd1', 'CREATE TABLE SIMPLE_DERBY_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ), + +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ), + +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user1\', \'passwd1\')' ), + +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ), + dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', 'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), @@ -30,6 +42,18 @@ SELECT dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2;create=true','user2','passwd2', 'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ), + +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ), + +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user2\', \'passwd2\')' ), + +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ), + dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', 'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), @@ -92,3 +116,5 @@ CREATE TABLE hive_table INSERT INTO hive_table VALUES(20); (SELECT * FROM ext_auth1 JOIN hive_table ON ext_auth1.ikey=hive_table.ikey) UNION ALL (SELECT * FROM ext_auth2 JOIN hive_table ON ext_auth2.ikey=hive_table.ikey); + +ALTER TABLE ext_auth1 SET TBLPROPERTIES ("hive.sql.dbcp.password.keystore" = "jceks://file/${system:test.tmp.dir}/../../../data/files/test.jceks"); diff --git a/ql/src/test/results/clientnegative/authorization_jdbc_keystore.q.out b/ql/src/test/results/clientnegative/authorization_jdbc_keystore.q.out new file mode 100644 index 0000000..0b8182a --- /dev/null +++ b/ql/src/test/results/clientnegative/authorization_jdbc_keystore.q.out @@ -0,0 +1 @@ +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_auth.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_auth.q.out index badc8b9..b299a38 100644 --- a/ql/src/test/results/clientpositive/llap/external_jdbc_auth.q.out +++ b/ql/src/test/results/clientpositive/llap/external_jdbc_auth.q.out @@ -12,6 +12,18 @@ SELECT 'CREATE TABLE SIMPLE_DERBY_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), #### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user1\', \'passwd1\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ), + +#### A masked pattern was here #### 'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), #### A masked pattern was here #### @@ -35,6 +47,18 @@ SELECT 'CREATE TABLE SIMPLE_DERBY_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), #### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user1\', \'passwd1\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ), + +#### A masked pattern was here #### 'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), #### A masked pattern was here #### @@ -50,7 +74,7 @@ limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -0 0 0 0 0 +0 0 0 0 0 0 0 0 0 PREHOOK: query: FROM src SELECT @@ -59,6 +83,18 @@ SELECT 'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), #### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user2\', \'passwd2\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ), + +#### A masked pattern was here #### 'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), #### A masked pattern was here #### @@ -82,6 +118,18 @@ SELECT 'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), #### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user2\', \'passwd2\')' ), + +#### A masked pattern was here #### +'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ), + +#### A masked pattern was here #### 'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), #### A masked pattern was here #### @@ -97,7 +145,7 @@ limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -0 0 0 0 0 +0 0 0 0 0 0 0 0 0 PREHOOK: query: CREATE EXTERNAL TABLE ext_auth1 ( ikey int, @@ -117,6 +165,7 @@ TBLPROPERTIES ( "hive.sql.dbcp.maxActive" = "1" ) PREHOOK: type: CREATETABLE +#### A masked pattern was here #### PREHOOK: Output: database:default PREHOOK: Output: default@ext_auth1 POSTHOOK: query: CREATE EXTERNAL TABLE ext_auth1 @@ -138,6 +187,7 @@ TBLPROPERTIES ( "hive.sql.dbcp.maxActive" = "1" ) POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### POSTHOOK: Output: database:default POSTHOOK: Output: default@ext_auth1 PREHOOK: query: CREATE EXTERNAL TABLE ext_auth2 @@ -159,6 +209,7 @@ TBLPROPERTIES ( "hive.sql.dbcp.maxActive" = "1" ) PREHOOK: type: CREATETABLE +#### A masked pattern was here #### PREHOOK: Output: database:default PREHOOK: Output: default@ext_auth2 POSTHOOK: query: CREATE EXTERNAL TABLE ext_auth2 @@ -180,6 +231,7 @@ TBLPROPERTIES ( "hive.sql.dbcp.maxActive" = "1" ) POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### POSTHOOK: Output: database:default POSTHOOK: Output: default@ext_auth2 PREHOOK: query: CREATE TABLE hive_table @@ -219,3 +271,13 @@ POSTHOOK: Input: default@hive_table #### A masked pattern was here #### 20 20 20.0 20.0 20 20 20 20.0 20.0 20 +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@ext_auth1 +#### A masked pattern was here #### +PREHOOK: Output: default@ext_auth1 +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@ext_auth1 +#### A masked pattern was here #### +POSTHOOK: Output: default@ext_auth1