diff --git accumulo-handler/pom.xml accumulo-handler/pom.xml index 1c18daf238..79fc809073 100644 --- accumulo-handler/pom.xml +++ accumulo-handler/pom.xml @@ -145,6 +145,13 @@ junit test + + org.apache.hive + hive-exec + ${project.version} + test-jar + test + diff --git itests/hcatalog-unit/pom.xml itests/hcatalog-unit/pom.xml index 28c9b6da24..ad59ebbe4a 100644 --- itests/hcatalog-unit/pom.xml +++ itests/hcatalog-unit/pom.xml @@ -331,6 +331,13 @@ 2.2 test + + org.apache.hive + hive-exec + ${project.version} + test-jar + test + diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/ManyMiniCluster.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/ManyMiniCluster.java index 0a90bc473d..fc3d500621 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/ManyMiniCluster.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/ManyMiniCluster.java @@ -29,12 +29,12 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; +import org.apache.hive.testutils.MiniZooKeeperCluster; import java.io.File; import java.io.IOException; diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZooKeeperTokenStore.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZooKeeperTokenStore.java index 0b4f2bc4f2..603155bf8f 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZooKeeperTokenStore.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZooKeeperTokenStore.java @@ -28,12 +28,12 @@ import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge.Server.ServerMode; import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; import org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport; +import org.apache.hive.testutils.MiniZooKeeperCluster; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.ACL; import org.junit.Assert; diff --git itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPrivilege.java itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPrivilege.java index 18b0b13478..de2e4937a8 100644 --- itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPrivilege.java +++ itests/hive-unit/src/test/java/org/apache/hive/service/server/TestInformationSchemaWithPrivilege.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Map; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hive.testutils.MiniZooKeeperCluster; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; diff --git itests/qtest-accumulo/pom.xml itests/qtest-accumulo/pom.xml index abfb5cb413..49ae58bf42 100644 --- itests/qtest-accumulo/pom.xml +++ itests/qtest-accumulo/pom.xml @@ -133,6 +133,13 @@ test tests + + org.apache.hive + hive-exec + ${project.version} + test-jar + test + junit @@ -177,34 +184,74 @@ hadoop-common ${hadoop.version} test - - - org.slf4j - slf4j-log4j12 - - - commons-logging - commons-logging - + + + org.slf4j + slf4j-log4j12 + + + commons-logging + commons-logging + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-test + + + org.apache.curator + curator-client + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-recipes + - + org.apache.hadoop hadoop-common ${hadoop.version} tests test - - - org.slf4j - slf4j-log4j12 - - - commons-logging - commons-logging - + + + org.slf4j + slf4j-log4j12 + + + commons-logging + commons-logging + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-test + + + org.apache.curator + curator-client + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-recipes + - + org.apache.hadoop hadoop-hdfs diff --git itests/qtest/pom.xml itests/qtest/pom.xml index e19a1b296f..c3117befb5 100644 --- itests/qtest/pom.xml +++ itests/qtest/pom.xml @@ -156,6 +156,13 @@ + + org.apache.hive + hive-exec + ${project.version} + test-jar + test + junit junit @@ -367,6 +374,15 @@ ${hbase.version} test + org.apache.tez tez-tests diff --git itests/util/pom.xml itests/util/pom.xml index bc14d1821f..8c14e544d8 100644 --- itests/util/pom.xml +++ itests/util/pom.xml @@ -204,6 +204,15 @@ ${tez.version} + junit junit diff --git itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java index 4e24e15717..5ad01a2dae 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java +++ itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java @@ -51,6 +51,10 @@ public AccumuloTestSetup() { @Override public void preTest(HiveConf conf) throws Exception { + System.setProperty("zookeeper.4lw.commands.whitelist", "*"); + System.setProperty("zookeeper.admin.enableServer", "false"); + System.setProperty("zookeeper.admin.serverAddress", "localhost"); + System.setProperty("zookeeper.admin.serverPort", "9080"); super.preTest(conf); setupWithHiveConf(conf); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java index 432d6eacfe..472b6cbe25 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java @@ -48,6 +48,10 @@ public void beforeClass() { String cleanupScript = cliConfig.getCleanupScript(); try { + System.setProperty("zookeeper.4lw.commands.whitelist", "*"); + System.setProperty("zookeeper.admin.enableServer", "false"); + System.setProperty("zookeeper.admin.serverAddress", "localhost"); + System.setProperty("zookeeper.admin.serverPort", "9080"); qt = new AccumuloQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, new AccumuloTestSetup(), initScript, cleanupScript); diff --git itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java index 9d0ffd8eca..3f8b766973 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java +++ itests/util/src/main/java/org/apache/hadoop/hive/llap/LlapItUtils.java @@ -25,7 +25,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hive.testutils.MiniZooKeeperCluster; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java index f8b7e29381..997b35e18f 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java @@ -41,7 +41,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; import org.apache.hadoop.hive.conf.HiveConf; @@ -62,6 +61,7 @@ import org.apache.hive.druid.MiniDruidCluster; import org.apache.hive.kafka.SingleNodeKafkaCluster; import org.apache.hive.kafka.Wikipedia; +import org.apache.hive.testutils.MiniZooKeeperCluster; import org.apache.logging.log4j.util.Strings; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; diff --git kafka-handler/pom.xml kafka-handler/pom.xml index 02f5a27506..6c82f82f76 100644 --- kafka-handler/pom.xml +++ kafka-handler/pom.xml @@ -110,7 +110,7 @@ org.apache.zookeeper zookeeper - 3.4.7 + 3.5.5 test @@ -192,4 +192,4 @@ - \ No newline at end of file + diff --git llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java index ef62b26123..7e3c20884b 100644 --- llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java +++ llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java @@ -50,6 +50,7 @@ @Before public void setUp() throws Exception { + hiveConf.set(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost"); registry = new LlapZookeeperRegistryImpl("TestLlapZookeeperRegistryImpl", hiveConf); server = new TestingServer(); diff --git llap-server/pom.xml llap-server/pom.xml index 2a9a9f178d..231aa2d9f0 100644 --- llap-server/pom.xml +++ llap-server/pom.xml @@ -240,6 +240,13 @@ test-jar test + + org.apache.hive + hive-exec + ${project.version} + test-jar + test + org.apache.hive hive-standalone-metastore-common diff --git llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java index 6af230e7b4..be51bf8c91 100644 --- llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java +++ llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java @@ -19,7 +19,6 @@ import java.io.File; import java.io.IOException; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -34,6 +33,7 @@ import org.apache.hadoop.service.Service; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hive.testutils.MiniZooKeeperCluster; import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; import com.google.common.base.Preconditions; diff --git pom.xml pom.xml index e061f64fc1..351df449d6 100644 --- pom.xml +++ pom.xml @@ -213,10 +213,10 @@ 1.4 1.5 2.9.1 - 3.4.6 + 3.5.5 1.1 2.4.0 - 2.12.0 + 4.2.0 3.0.0 0.6.0 2.2.4 @@ -509,6 +509,20 @@ org.apache.hive hive-storage-api ${storage-api.version} + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-client + + + org.apache.curator + curator-recipes + + org.apache.pig @@ -703,6 +717,18 @@ commons-logging commons-logging + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-test + @@ -726,6 +752,22 @@ org.apache.httpcomponents httpclient + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-test + + + org.apache.curator + curator-client + + + org.apache.curator + curator-recipes + diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java index b001d027cc..d0425e5572 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java @@ -86,7 +86,9 @@ public void testGetSplitLocationProvider() throws IOException, URISyntaxExceptio LlapServiceInstance inactive = new InactiveServiceInstance(INACTIVE); instances.add(inactive); - LlapZookeeperRegistryImpl dynRegistry = new LlapZookeeperRegistryImpl("dyn", new HiveConf()); + HiveConf conf = new HiveConf(); + conf.set(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost"); + LlapZookeeperRegistryImpl dynRegistry = new LlapZookeeperRegistryImpl("dyn", conf); Endpoint rpcEndpoint = RegistryTypeUtils.ipcEndpoint("llap", new InetSocketAddress(ACTIVE, 4000)); Endpoint shuffle = RegistryTypeUtils.ipcEndpoint("shuffle", new InetSocketAddress(ACTIVE, 4000)); Endpoint mng = RegistryTypeUtils.ipcEndpoint("llapmng", new InetSocketAddress(ACTIVE, 4000)); diff --git ql/src/test/org/apache/hive/testutils/MiniZooKeeperCluster.java ql/src/test/org/apache/hive/testutils/MiniZooKeeperCluster.java index 646c1c1159..eec628263a 100644 --- ql/src/test/org/apache/hive/testutils/MiniZooKeeperCluster.java +++ ql/src/test/org/apache/hive/testutils/MiniZooKeeperCluster.java @@ -49,7 +49,7 @@ * *XXX: copied from the only used class by qtestutil from hbase-tests */ -class MiniZooKeeperCluster { +public class MiniZooKeeperCluster { private static final Logger LOG = LoggerFactory.getLogger(MiniZooKeeperCluster.class); private static final int TICK_TIME = 2000; @@ -384,7 +384,7 @@ public void killOneBackupZooKeeperServer() throws IOException, InterruptedExcept } // XXX: From o.a.zk.t.ClientBase - private static boolean waitForServerDown(int port, long timeout) throws IOException { + public static boolean waitForServerDown(int port, long timeout) throws IOException { long start = System.currentTimeMillis(); while (true) { try { @@ -413,7 +413,7 @@ private static boolean waitForServerDown(int port, long timeout) throws IOExcept } // XXX: From o.a.zk.t.ClientBase - private static boolean waitForServerUp(int port, long timeout) throws IOException { + public static boolean waitForServerUp(int port, long timeout) throws IOException { long start = System.currentTimeMillis(); while (true) { try { diff --git standalone-metastore/metastore-common/pom.xml standalone-metastore/metastore-common/pom.xml index 667905318e..95e53b9502 100644 --- standalone-metastore/metastore-common/pom.xml +++ standalone-metastore/metastore-common/pom.xml @@ -113,6 +113,10 @@ commons-logging commons-logging + + org.apache.curator + curator-framework + @@ -194,6 +198,43 @@ commons-logging commons-logging + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + org.slf4j + slf4j-log4j12 + + + commons-logging + commons-logging + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpclient + + + org.jboss.netty + netty + + + + + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.curator + curator-framework + ${curator.version} + diff --git standalone-metastore/metastore-server/pom.xml standalone-metastore/metastore-server/pom.xml index b8e3f31bc9..0ff1b32911 100644 --- standalone-metastore/metastore-server/pom.xml +++ standalone-metastore/metastore-server/pom.xml @@ -119,6 +119,10 @@ commons-logging commons-logging + + org.apache.curator + curator-framework + diff --git standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml index f54b1617fb..59b61e15a7 100644 --- standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml +++ standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml @@ -37,10 +37,38 @@ metastore-tools-common ${hive.version} compile + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-client + + org.apache.hive.hcatalog hive-hcatalog-server-extensions + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-client + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-recipes + + diff --git standalone-metastore/metastore-tools/pom.xml standalone-metastore/metastore-tools/pom.xml index fd2498788a..63f2369a42 100644 --- standalone-metastore/metastore-tools/pom.xml +++ standalone-metastore/metastore-tools/pom.xml @@ -54,11 +54,39 @@ org.apache.hive.hcatalog hive-hcatalog-server-extensions ${hive.version} + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-client + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-recipes + + org.apache.hive hive-common ${hive.version} + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-client + + org.apache.hive diff --git standalone-metastore/pom.xml standalone-metastore/pom.xml index c0c1536d80..3de774a0d0 100644 --- standalone-metastore/pom.xml +++ standalone-metastore/pom.xml @@ -94,7 +94,8 @@ 1.3.0 2.7.0-SNAPSHOT 1.3 - 2.12.0 + 4.2.0 + 3.5.5 you-must-set-this-to-run-thrift @@ -186,6 +187,28 @@ org.apache.hadoop hadoop-common ${hadoop.version} + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-test + + + org.apache.curator + curator-client + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-recipes + + org.apache.hadoop diff --git storage-api/pom.xml storage-api/pom.xml index 2f180a48c7..bb1ec1f81f 100644 --- storage-api/pom.xml +++ storage-api/pom.xml @@ -89,6 +89,22 @@ org.slf4j slf4j-log4j12 + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-client + + + org.apache.curator + curator-recipes + diff --git upgrade-acid/pre-upgrade/pom.xml upgrade-acid/pre-upgrade/pom.xml index 818487a44a..3abe59675d 100644 --- upgrade-acid/pre-upgrade/pom.xml +++ upgrade-acid/pre-upgrade/pom.xml @@ -71,6 +71,14 @@ tomcat jasper-runtime + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-framework + @@ -78,12 +86,40 @@ hive-exec ${hdp.hive.version} provided + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-framework + + org.apache.hadoop hadoop-common ${hdp.hadoop.version} provided + + + org.apache.zookeeper + zookeeper + + + org.apache.curator + curator-client + + + org.apache.curator + curator-recipes + + + org.apache.curator + curator-framework + +