From 7a25663b64d3d4c869fb3f1055e485e01168b179 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Mon, 4 Dec 2017 13:59:12 -0800 Subject: [PATCH] HBASE-19421 branch-1 does not compile against Hadoop 3.0.0 --- hbase-client/pom.xml | 8 +- hbase-common/pom.xml | 58 ++++- hbase-examples/pom.xml | 107 +++++---- hbase-external-blockcache/pom.xml | 12 +- hbase-it/pom.xml | 14 +- hbase-prefix-tree/pom.xml | 8 +- hbase-procedure/pom.xml | 8 +- .../src/main/resources/supplemental-models.xml | 262 ++++++++++++++++++++- hbase-rest/pom.xml | 20 ++ hbase-rsgroup/pom.xml | 12 +- hbase-server/pom.xml | 18 +- .../org/apache/hadoop/hbase/http/HttpServer.java | 9 +- .../hadoop/hbase/regionserver/HRegionServer.java | 2 +- .../org/apache/hadoop/hbase/util/MBeanUtil.java | 83 +++++++ .../hadoop/hbase/master/MockRegionServer.java | 2 - .../hadoop/hbase/regionserver/TestBulkLoad.java | 2 +- hbase-shell/pom.xml | 18 +- hbase-testing-util/pom.xml | 20 +- hbase-thrift/pom.xml | 17 +- .../hadoop/hbase/thrift/IncrementCoalescer.java | 2 +- pom.xml | 71 +++++- 21 files changed, 667 insertions(+), 86 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/util/MBeanUtil.java diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index 68ebf28403..32bc7219dd 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -358,7 +358,7 @@ - 3.0-SNAPSHOT + ${hadoop-three.version} @@ -368,6 +368,12 @@ org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 50f6824e4d..e154434825 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -484,12 +484,68 @@ - 3.0-SNAPSHOT + ${hadoop-three.version} org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + javax.servlet + servlet-api + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-client + + + com.sun.jersey + jersey-json + + + com.sun.jersey.contribs + jersey-guice + + + com.google.inject + guice + + + com.google.inject.extensions + guice-servlet + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml index 138ea41c71..89e134684c 100644 --- a/hbase-examples/pom.xml +++ b/hbase-examples/pom.xml @@ -221,61 +221,64 @@ if we can combine these profiles somehow --> profile for building against Hadoop 3.0.x. Activate using: mvn -Dhadoop.profile=3.0 --> - - hadoop-3.0 - - - hadoop.profile - 3.0 - - - - 3.0-SNAPSHOT - - - - org.apache.hadoop - hadoop-common - - - org.apache.hadoop - hadoop-annotations - - - jdk.tools - jdk.tools - - - - - org.apache.hadoop - hadoop-minicluster - - - - - - maven-dependency-plugin - - - create-mrapp-generated-classpath - generate-test-resources - - build-classpath - - - - ${project.build.directory}/test-classes/mrapp-generated-classpath - - - - - - - + ${project.build.directory}/test-classes/mrapp-generated-classpath + + + + + + + + compile-protobuf diff --git a/hbase-external-blockcache/pom.xml b/hbase-external-blockcache/pom.xml index 165f2b79eb..c58dbd6aa3 100644 --- a/hbase-external-blockcache/pom.xml +++ b/hbase-external-blockcache/pom.xml @@ -280,12 +280,22 @@ - 3.0-SNAPSHOT + ${hadoop-three.version} org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + + com.google.guava + guava + + diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index dc7cf44735..56c70efead 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -387,16 +387,28 @@ - 3.0-SNAPSHOT + ${hadoop-three.version} org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + org.apache.hadoop hadoop-minicluster + + + org.apache.htrace + htrace-core + + org.apache.hadoop diff --git a/hbase-prefix-tree/pom.xml b/hbase-prefix-tree/pom.xml index 9f7d9ca547..e48f34f02f 100644 --- a/hbase-prefix-tree/pom.xml +++ b/hbase-prefix-tree/pom.xml @@ -223,12 +223,18 @@ - 3.0-SNAPSHOT + ${hadoop-three.version} org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + diff --git a/hbase-procedure/pom.xml b/hbase-procedure/pom.xml index 55e1b6c93b..98af96f770 100644 --- a/hbase-procedure/pom.xml +++ b/hbase-procedure/pom.xml @@ -182,12 +182,18 @@ - 3.0-SNAPSHOT + ${hadoop-three.version} org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml b/hbase-resource-bundle/src/main/resources/supplemental-models.xml index 002d82d8d7..bb24b0c3d6 100644 --- a/hbase-resource-bundle/src/main/resources/supplemental-models.xml +++ b/hbase-resource-bundle/src/main/resources/supplemental-models.xml @@ -61,15 +61,152 @@ under the License. + - commons-beanutils - commons-beanutils-core + org.eclipse.jetty + jetty-http + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-io + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-jmx + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-server + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-servlet + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-util + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-util-ajax + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-webapp + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-xml + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-jsp + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty + jetty-security - - The Apache Software Foundation - http://www.apache.org/ - Apache License, Version 2.0 @@ -79,7 +216,62 @@ under the License. - + + + org.eclipse.jetty.orbit + javax.servlet.jsp.jstl + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + + org.eclipse.jetty.orbit + org.eclipse.jdt.core + + + + Eclipse Public License 1.0 + http://www.eclipse.org/legal/epl-v10.html + repo + + + + + + + org.eclipse.jetty.toolchain + jetty-schemas + + + Common Development and Distribution License (CDDL) v1.0 + https://glassfish.dev.java.net/public/CDDLv1.0.html + repo + + + + + + + + org.javassist + javassist + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + com.github.stephenc.findbugs @@ -575,6 +767,32 @@ under the License. + javax.servlet + javax.servlet-api + + + Common Development and Distribution License (CDDL) v1.0 + https://glassfish.dev.java.net/public/CDDLv1.0.html + repo + + + + + + + javax.servlet.jsp + javax.servlet.jsp-api + + + CDDL 1.1 + https://glassfish.java.net/public/CDDL+GPL_1_1.html + repo + + + + + + javax.inject javax.inject @@ -1413,6 +1631,23 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + org.codehaus.woodstox + stax2-api + 3.1.4 + + + BSD license + https://opensource.org/licenses/bsd-license.php + repo + +Copyright 2010 FasterXML.com + + + + + + org.jruby.jcodings jcodings @@ -1833,6 +2068,19 @@ Copyright 2005 Sun Microsystems, Inc. and portions Copyright Apache Software Fou + javax.ws.rs + jsr311-api + + + Common Development and Distribution License (CDDL) v1.0 + http://www.opensource.org/licenses/cddl1.php + repo + + + + + + org.jamon jamon-runtime Jamon runtime support classes diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml index 688f38a09b..a948608f6a 100644 --- a/hbase-rest/pom.xml +++ b/hbase-rest/pom.xml @@ -425,15 +425,35 @@ 3.0 + + ${hadoop-three.version} + org.apache.hadoop hadoop-mapreduce-client-core test + + + com.google.guava + guava + + + + + org.apache.hadoop + hadoop-annotations + ${hadoop-three.version} org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + org.apache.hadoop diff --git a/hbase-rsgroup/pom.xml b/hbase-rsgroup/pom.xml index 2bb695800f..e90af2b106 100644 --- a/hbase-rsgroup/pom.xml +++ b/hbase-rsgroup/pom.xml @@ -261,16 +261,18 @@ - 3.0-SNAPSHOT + ${hadoop-three.version} org.apache.hadoop - hadoop-auth - - - org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 16b2f9eff1..7346f1a53e 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -786,17 +786,33 @@ org.apache.hadoop + hadoop-distcp + ${hadoop-three.version} + + + org.apache.hadoop hadoop-common org.apache.hadoop + hadoop-annotations + ${hadoop-three.version} + + + org.apache.hadoop hadoop-minicluster + + + com.google.guava + guava + + io.netty netty - 3.6.2.Final + ${netty.hadoop.version} test diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index c630dc0da7..a8ed8a4326 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -59,7 +59,6 @@ import org.apache.hadoop.hbase.http.jmx.JMXJsonServlet; import org.apache.hadoop.hbase.http.log.LogLevel; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.ReflectionUtils; -import org.apache.hadoop.metrics.MetricsServlet; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; @@ -710,9 +709,15 @@ public class HttpServer implements FilterContainer { // set up default servlets addServlet("stacks", "/stacks", StackServlet.class); addServlet("logLevel", "/logLevel", LogLevel.Servlet.class); - addServlet("metrics", "/metrics", MetricsServlet.class); addServlet("jmx", "/jmx", JMXJsonServlet.class); addServlet("conf", "/conf", ConfServlet.class); + try { + Class clazz = (Class) + Class.forName("org.apache.hadoop.metrics.MetricsServlet"); + addServlet("metrics", "/metrics", clazz); + } catch (Exception e) { + LOG.warn("MetricsServlet class not found, metrics servlet will not start", e); + } } public void addContext(Context ctxt, boolean isFiltered) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 3a731a715f..1cb2e4685a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -166,6 +166,7 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HasThread; import org.apache.hadoop.hbase.util.JSONBean; import org.apache.hadoop.hbase.util.JvmPauseMonitor; +import org.apache.hadoop.hbase.util.MBeanUtil; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.hadoop.hbase.util.Sleeper; import org.apache.hadoop.hbase.util.Threads; @@ -183,7 +184,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.metrics.util.MBeanUtil; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.zookeeper.KeeperException; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MBeanUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MBeanUtil.java new file mode 100644 index 0000000000..710ff26ae8 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MBeanUtil.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.lang.management.ManagementFactory; + +import javax.management.InstanceNotFoundException; +import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.management.InstanceAlreadyExistsException; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +@InterfaceAudience.Private +public class MBeanUtil { + + /** + * Register the MBean using our standard MBeanName format + * "hadoop:service=,name=" + * Where the and are the supplied parameters + * + * @param serviceName + * @param nameName + * @param theMbean - the MBean to register + * @return the named used to register the MBean + */ + static public ObjectName registerMBean(final String serviceName, + final String nameName, + final Object theMbean) { + final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName name = getMBeanName(serviceName, nameName); + try { + mbs.registerMBean(theMbean, name); + return name; + } catch (InstanceAlreadyExistsException ie) { + // Ignore if instance already exists + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + static public void unregisterMBean(ObjectName mbeanName) { + final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + if (mbeanName == null) + return; + try { + mbs.unregisterMBean(mbeanName); + } catch (InstanceNotFoundException e ) { + // ignore + } catch (Exception e) { + e.printStackTrace(); + } + } + + static private ObjectName getMBeanName(final String serviceName, + final String nameName) { + ObjectName name = null; + try { + name = new ObjectName("hadoop:" + + "service=" + serviceName + ",name=" + nameName); + } catch (MalformedObjectNameException e) { + e.printStackTrace(); + } + return name; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index 14bd2fde34..42c04e4480 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -115,8 +115,6 @@ import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; -import net.sf.ehcache.search.Results; - /** * A mock RegionServer implementation. * Use this when you can't bend Mockito to your liking (e.g. return null result diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java index 735fec8fc8..5649d8acb1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java @@ -272,7 +272,7 @@ public class TestBulkLoad { HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(conf); // TODO We need a way to do this without creating files File hFileLocation = testFolder.newFile(); - FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation)); + FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null); try { hFileFactory.withOutputStream(out); hFileFactory.withFileContext(new HFileContext()); diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml index 44b6095b0b..3f817837c7 100644 --- a/hbase-shell/pom.xml +++ b/hbase-shell/pom.xml @@ -395,16 +395,32 @@ - 3.0-SNAPSHOT + ${hadoop-three.version} org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + org.apache.hadoop hadoop-minicluster + + + org.apache.htrace + htrace-core + + + com.google.guava + guava + + org.apache.hadoop diff --git a/hbase-testing-util/pom.xml b/hbase-testing-util/pom.xml index 4443723d23..5adf241ba0 100644 --- a/hbase-testing-util/pom.xml +++ b/hbase-testing-util/pom.xml @@ -195,25 +195,37 @@ 3.0 - - 3.0-SNAPSHOT - + + ${hadoop-three.version} + org.apache.hadoop hadoop-common compile + + + org.apache.htrace + htrace-core + + org.apache.hadoop hadoop-minicluster compile + + + org.apache.htrace + htrace-core + + org.apache.hadoop hadoop-minikdc - + diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml index c46b9397f1..a03e129b2e 100644 --- a/hbase-thrift/pom.xml +++ b/hbase-thrift/pom.xml @@ -565,16 +565,28 @@ - 3.0-SNAPSHOT + ${hadoop-three.version} org.apache.hadoop hadoop-common + + + org.apache.htrace + htrace-core + + org.apache.hadoop hadoop-minicluster + + + org.apache.htrace + htrace-core + + @@ -593,7 +605,8 @@ the required classpath that is required in the env of the launch container in the mini mr/yarn cluster --> - ${project.build.directory}/test-classes/mrapp-generated-classpath + ${project.build.directory}/test-classes/mrapp-generated-classpath + diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java index 205e3ad579..c8ef247628 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/IncrementCoalescer.java @@ -39,8 +39,8 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler; import org.apache.hadoop.hbase.thrift.generated.TIncrement; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.MBeanUtil; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.metrics.util.MBeanUtil; import org.apache.thrift.TException; /** diff --git a/pom.xml b/pom.xml index 1f331d8dd3..70e1947ac9 100644 --- a/pom.xml +++ b/pom.xml @@ -2360,6 +2360,18 @@ io.netty netty + + javax.inject + javax.inject + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + @@ -2391,6 +2403,10 @@ hadoop-hdfs + org.apache.htrace + htrace-core + + javax.servlet.jsp jsp-api @@ -2406,6 +2422,14 @@ xerces xercesImpl + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + ${hadoop-three.version} @@ -2417,6 +2441,10 @@ test + org.apache.htrace + htrace-core + + javax.servlet.jsp jsp-api @@ -2432,12 +2460,26 @@ xerces xercesImpl + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + org.apache.hadoop hadoop-auth ${hadoop-three.version} + + + com.google.guava + guava + + org.apache.hadoop @@ -2445,6 +2487,14 @@ ${hadoop-three.version} + org.apache.htrace + htrace-core + + + commons-beanutils + commons-beanutils + + javax.servlet.jsp jsp-api @@ -2464,6 +2514,18 @@ com.google.code.findbugs jsr305 + + junit + junit + + + org.codehause.jackson + jackson-core-asl + + + org.codehause.jackson + jackson-mapper-asl + @@ -2484,6 +2546,14 @@ ${hadoop-three.version} + org.apache.htrace + htrace-core + + + commons-httpclient + commons-httpclient + + javax.servlet.jsp jsp-api @@ -2514,7 +2584,6 @@ - -- 2.13.4