commit 808ffeac95185bbdb8484baf59bb8adc897e6503 Author: Janaki Lahorani Date: Wed Jan 24 14:02:12 2018 -0800 HIVE-18586: Change Derby Version to 10.14.1.0 HCatalog tests use Security Manager to handle exits. With Derby version 10.14.1, if a security manager is configured, embedded Derby requires usederbyinternals permission, and that is checked directly using AccessController.checkPermission. DerbyPolicy class is used to setup a security policy to grant usederbyinternals, in tests that use NoExitSecurityManager. NoExitSecurityManager is removed from tests that don't need the same. diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java new file mode 100644 index 0000000000000000000000000000000000000000..cecf6dc676f75dca0500d0047027898e3a35570f --- /dev/null +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/DerbyPolicy.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.hcatalog; + +import org.apache.derby.security.SystemPermission; + +import java.security.CodeSource; +import java.security.Permission; +import java.security.PermissionCollection; +import java.security.Policy; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.Iterator; + +/** + * A security policy that grants usederbyinternals + * + *

+ * HCatalog tests use Security Manager to handle exits. With Derby version 10.14.1, if a + * security manager is configured, embedded Derby requires usederbyinternals permission, and + * that is checked directly using AccessController.checkPermission. This class will be used to + * setup a security policy to grant usederbyinternals, in tests that use NoExitSecurityManager. + *

+ */ +public class DerbyPolicy extends Policy { + + private static PermissionCollection perms; + + public DerbyPolicy() { + super(); + if (perms == null) { + perms = new DerbyPermissionCollection(); + addPermissions(); + } + } + + @Override + public PermissionCollection getPermissions(CodeSource codesource) { + return perms; + } + + private void addPermissions() { + SystemPermission systemPermission = new SystemPermission("engine", "usederbyinternals"); + perms.add(systemPermission); + } + + class DerbyPermissionCollection extends PermissionCollection { + + ArrayList perms = new ArrayList(); + + public void add(Permission p) { + perms.add(p); + } + + public boolean implies(Permission p) { + for (Iterator i = perms.iterator(); i.hasNext();) { + if (((Permission) i.next()).implies(p)) { + return true; + } + } + return false; + } + + public Enumeration elements() { + return Collections.enumeration(perms); + } + + public boolean isReadOnly() { + return false; + } + } +} + diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 4dbf7acb9d569dd608b83ca7db0f6e18d0c3d02b..a2c05baa6426a3c4bc7ee01492e95829872b37ed 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -19,6 +19,7 @@ package org.apache.hive.hcatalog.cli; import java.io.FileNotFoundException; +import java.security.Policy; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -50,6 +51,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hive.hcatalog.DerbyPolicy; import org.apache.hive.hcatalog.ExitException; import org.apache.hive.hcatalog.NoExitSecurityManager; import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; @@ -85,6 +87,7 @@ protected void setUp() throws Exception { securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); + Policy.setPolicy(new DerbyPolicy()); hcatConf = new HiveConf(this.getClass()); hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://127.0.0.1:" + msPort); diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index 7205a7902bfb7159247caa3d2c023853b7bbae17..fb6a7f47028f96957c876060de07a4012f45cc82 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; +import java.security.Policy; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -55,6 +56,7 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hive.hcatalog.DerbyPolicy; import org.apache.hive.hcatalog.NoExitSecurityManager; import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; import org.apache.hive.hcatalog.data.DefaultHCatRecord; @@ -110,6 +112,7 @@ public static void setup() throws Exception { isServerRunning = true; securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); + Policy.setPolicy(new DerbyPolicy()); hcatConf = new HiveConf(TestHCatPartitionPublish.class); hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/package-info.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/package-info.java new file mode 100644 index 0000000000000000000000000000000000000000..a1c304389e63a3210057cd0397bb8402a4c291b5 --- /dev/null +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A package info file for hcatalog tests. + */ +package org.apache.hive.hcatalog; diff --git hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index 2a36c85cc4e3fc1ecdc5af97e02a33e0a90b15b5..515eef25f8f6d39afe16dab98fca169a15429808 100644 --- hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.math.BigInteger; +import java.security.Policy; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -52,6 +53,7 @@ import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hive.hcatalog.DerbyPolicy; import org.apache.hive.hcatalog.api.repl.Command; import org.apache.hive.hcatalog.api.repl.ReplicationTask; import org.apache.hive.hcatalog.api.repl.ReplicationUtils; @@ -121,6 +123,8 @@ public static void startMetaStoreServer() throws Exception { msPort = MetaStoreTestUtils.startMetaStore(); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); + Policy.setPolicy(new DerbyPolicy()); + hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); diff --git pom.xml pom.xml index bd19ca38a0a36feb813a0d52a264c98122de8c44..1683c59dad032bb18e08ff4763bfc9af95a01ce9 100644 --- pom.xml +++ pom.xml @@ -137,7 +137,7 @@ 3.2 1.5.4 1.4 - 10.11.1.1 + 10.14.1.0 3.1.0 0.1.2 0.11.0 diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java index ba86e052ed2d38532bca84f8f487f59403739a5b..3a0c103fb44a6e861b615c69a72e5eadd694d135 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.metastore; -import java.security.Permission; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -49,25 +47,6 @@ private HiveMetaStoreClient hmsc; - public static class NoExitSecurityManager extends SecurityManager { - - @Override - public void checkPermission(Permission perm) { - // allow anything. - } - - @Override - public void checkPermission(Permission perm, Object context) { - // allow anything. - } - - @Override - public void checkExit(int status) { - super.checkExit(status); - throw new RuntimeException("System.exit() was called. Raising exception."); - } - } - @AfterClass public static void tearDown() throws Exception { LOG.info("Shutting down metastore."); @@ -78,7 +57,7 @@ public static void tearDown() throws Exception { public static void startMetaStoreServer() throws Exception { securityManager = System.getSecurityManager(); - System.setSecurityManager(new NoExitSecurityManager()); + System.setSecurityManager(null); Configuration metastoreConf = MetastoreConf.newMetastoreConf(); MetastoreConf.setClass(metastoreConf, ConfVars.EXPRESSION_PROXY_CLASS, MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java index 57e5a4126e2ac28e990ba8f3be525c91ebe68dad..19560a6506f9f5521f3958d4e7e69024e7bb1f78 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java @@ -37,7 +37,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.security.Permission; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -54,27 +53,6 @@ private static Configuration conf; private static SecurityManager securityManager; - public static class NoExitSecurityManager extends SecurityManager { - - @Override - public void checkPermission(Permission perm) { - // allow anything. - } - - @Override - public void checkPermission(Permission perm, Object context) { - // allow anything. - } - - @Override - public void checkExit(int status) { - - super.checkExit(status); - throw new RuntimeException("System.exit() was called. Raising exception. "); - } - } - - @AfterClass public static void tearDown() throws Exception { LOG.info("Shutting down metastore."); @@ -93,7 +71,7 @@ public static void startMetaStoreServer() throws Exception { MetaStoreTestUtils.setConfForStandloneMode(metastoreConf); msPort = MetaStoreTestUtils.startMetaStore(metastoreConf); securityManager = System.getSecurityManager(); - System.setSecurityManager(new NoExitSecurityManager()); + System.setSecurityManager(null); conf = MetastoreConf.newMetastoreConf(); MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + msPort); MetastoreConf.setLongVar(conf, ConfVars.THRIFT_CONNECTION_RETRIES, 3);