From 5ee121e1897f619ee43a04756ff4b9dbcd81fbe2 Mon Sep 17 00:00:00 2001 From: Matt Warhaftig Date: Thu, 9 Jun 2016 11:30:09 -0400 Subject: [PATCH] HBASE-15112 Allow coprocessors to add attribute info via SPI. --- .../apache/hadoop/hbase/CoprocessorAttribute.java | 101 ++++++++ .../apache/hadoop/hbase/CoprocessorAttributes.java | 41 ++++ .../hbase/tmpl/master/MasterStatusTmpl.jamon | 11 + .../hbase/tmpl/regionserver/RSStatusTmpl.jamon | 11 + .../coprocessor/CoprocessorAttributesService.java | 115 +++++++++ .../hadoop/hbase/coprocessor/CoprocessorHost.java | 19 ++ .../org/apache/hadoop/hbase/master/HMaster.java | 10 + .../apache/hadoop/hbase/master/MasterServices.java | 7 + .../hadoop/hbase/regionserver/HRegionServer.java | 25 ++ .../coprocessor/TestCoprocessorAttributes.java | 267 +++++++++++++++++++++ .../hbase/master/MockNoopMasterServices.java | 7 + .../hbase/master/TestMasterStatusServlet.java | 12 + .../hadoop/hbase/master/TestTableLockManager.java | 14 +- .../hbase/regionserver/TestRSStatusServlet.java | 12 + .../org.apache.hadoop.hbase.CoprocessorAttributes | 21 ++ 15 files changed, 672 insertions(+), 1 deletion(-) create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorAttribute.java create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorAttributes.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorAttributesService.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorAttributes.java create mode 100644 hbase-server/src/test/resources/META-INF/services/org.apache.hadoop.hbase.CoprocessorAttributes diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorAttribute.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorAttribute.java new file mode 100644 index 0000000..5893f59 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorAttribute.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * Coprocessor attribute instance. Contains a single attribute available via the HBase Web UI's + * 'Software Attributes' table and API. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public class CoprocessorAttribute { + private String name; + private String value; + private String description; + + /** + * Construct CoprocessorAttribute with just name and value. + * @param name + * @param value + */ + public CoprocessorAttribute(String name, String value) { + this(name, value, ""); + } + + /** + * Construct CoprocessorAttribute with name, value, and description. + * @param name + * @param value + */ + public CoprocessorAttribute(String name, String value, String description) { + this.name = name != null ? name : ""; + this.value = value != null ? value : ""; + this.description = description != null ? description : ""; + } + + /** + * @return attribute name + */ + public String getName() { + return name; + } + + /** + * Set attribute name. + * @param name + */ + public void setName(String name) { + this.name = name != null ? name : ""; + } + + /** + * @return attribute value + */ + public String getValue() { + return value; + } + + /** + * Set attribute value. + * @param value + */ + public void setValue(String value) { + this.value = value != null ? value : ""; + } + + /** + * @return attribute description + */ + public String getDescription() { + return description; + } + + /** + * Set attribute description. + * @param description + */ + public void setDescription(String description) { + this.description = description != null ? description : ""; + } + +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorAttributes.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorAttributes.java new file mode 100644 index 0000000..de3a78f --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorAttributes.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase; + +import java.util.List; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * Coprocessor attributes Service Provider Interface (SPI) for publishing coprocessor attributes to + * the HBase Web UI's 'Software Attributes' table and API. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface CoprocessorAttributes { + + /** + * Implementing this method will publish the returned coprocessor attributes. + * @return List of {@link CoprocessorAttribute}. + */ + List coprocessorAttributes(); + +} diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index e39ad8a..303baad 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -54,6 +54,7 @@ org.apache.hadoop.hbase.util.Bytes; org.apache.hadoop.hbase.util.FSUtils; org.apache.hadoop.hbase.util.JvmVersion; org.apache.hadoop.util.StringUtils; +org.apache.hadoop.hbase.CoprocessorAttribute; <%if format.equals("json") %> @@ -329,6 +330,16 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); LoadBalancer to be used in the Master + <%for String coproc : master.getMasterCoprocessorAttributes().keySet()%> + <%for CoprocessorAttribute att : master.getMasterCoprocessorAttributes() + .get(coproc) %> + + <% coproc %> - <% att.getName() %> + <% att.getValue() %> + <% att.getDescription() %> + + + diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon index 21e9b9f..0e68151 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -32,6 +32,7 @@ org.apache.hadoop.hbase.HBaseConfiguration; org.apache.hadoop.hbase.protobuf.ProtobufUtil; org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; +org.apache.hadoop.hbase.CoprocessorAttribute; <%doc>If json AND bcn is NOT an empty string presume it a block cache view request. <%if format.equals("json") && bcn != null && bcn.length() > 0 %> @@ -201,6 +202,16 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; Address of HBase Master + <%for String coproc : regionServer.getRegionServerCoprocessorAttributes().keySet() %> + <%for CoprocessorAttribute att : regionServer.getRegionServerCoprocessorAttributes() + .get(coproc) %> + + <% coproc %> - <% att.getName() %> + <% att.getValue() %> + <% att.getDescription() %> + + + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorAttributesService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorAttributesService.java new file mode 100644 index 0000000..0119dca --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorAttributesService.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorAttribute; +import org.apache.hadoop.hbase.CoprocessorAttributes; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +/** + * Service to gather coprocessor attributes from SPI. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +class CoprocessorAttributesService { + + private static CoprocessorAttributesService service; + private ServiceLoader loader; + private static final Log LOG = LogFactory.getLog(CoprocessorAttributesService.class); + + private LoadingCache, List> coprocessAttCache = + CacheBuilder.newBuilder().maximumSize(100).expireAfterAccess(5, TimeUnit.MINUTES) + .build(new CacheLoader, List>() { + @Override + public List load(Class coprocClass) throws Exception { + return loadAttributes(coprocClass); + } + }); + + private CoprocessorAttributesService() { + loader = ServiceLoader.load(CoprocessorAttributes.class); + } + + public static synchronized CoprocessorAttributesService getInstance() { + if (service == null) { + service = new CoprocessorAttributesService(); + } + return service; + } + + /** + * Get the coprocessor attributes associated with a list of coprocessor classes. + */ + public Map> getAttributes(List> coprocClasses) { + Map> results = + new HashMap>(); + try { + for (Class coprocClass : coprocClasses) { + results.put(coprocClass.getSimpleName(), coprocessAttCache.get(coprocClass)); + } + } catch (ExecutionException ee) { + LOG.error("Error getting coprocessor attributes cache.", ee); + } + return results; + } + + private List loadAttributes(Class coprocClass) { + URL[] url = new URL[] { coprocClass.getProtectionDomain().getCodeSource().getLocation() }; + + // Use custom ClassLoader because dynamically loaded coprocessors are otherwise unavailable. + ClassLoader cl = new URLClassLoader(url); + loader = ServiceLoader.load(CoprocessorAttributes.class, cl); + + try { + Iterator attItr = loader.iterator(); + while (attItr.hasNext()) { + CoprocessorAttributes atts = attItr.next(); + if (atts.getClass().getSimpleName().equals(coprocClass.getSimpleName())) { + // Guava does not like nulls so use empty list instead. + return atts.coprocessorAttributes() != null ? atts.coprocessorAttributes() + : new ArrayList(); + } + } + } catch (ServiceConfigurationError serviceError) { + LOG.error("Error getting coprocessor attributes.", serviceError); + } + + return new ArrayList(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java index da0e8b1..e9d2c22 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java @@ -23,8 +23,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.UUID; @@ -40,6 +42,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.CoprocessorAttribute; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseInterfaceAudience; @@ -127,6 +130,22 @@ public abstract class CoprocessorHost { } /** + * Used to gather coprocessors attributes of each active coprocessor class. (HBASE-15112: Allow + * coprocessors to extend 'software attributes' list). + */ + public Map> getCoprocessorAttributes() { + List> coprocClasses = new ArrayList>(); + for (CoprocessorEnvironment e: coprocessors) { + coprocClasses.add(e.getInstance().getClass()); + } + + Map> map = new HashMap>(); + map.putAll(CoprocessorAttributesService.getInstance().getAttributes(coprocClasses)); + + return map; + } + + /** * Load system coprocessors once only. Read the class names from configuration. * Called by constructor. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index f8d0003..6e4c4f7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -60,6 +60,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.CoordinatedStateException; import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.CoprocessorAttribute; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HBaseInterfaceAudience; @@ -2250,6 +2251,15 @@ public class HMaster extends HRegionServer implements MasterServices { } @Override + public Map> getMasterCoprocessorAttributes() { + // Prevent NPE if Web UI visited before MasterCoprocessorHost is initialized. + if (getMasterCoprocessorHost() == null) { + return Maps.newHashMap(); + } + return getMasterCoprocessorHost().getCoprocessorAttributes(); + } + + @Override public MasterQuotaManager getMasterQuotaManager() { return quotaManager; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index 21f14e8..f921b00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -20,7 +20,9 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; import java.util.List; +import java.util.Map; +import org.apache.hadoop.hbase.CoprocessorAttribute; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; @@ -103,6 +105,11 @@ public interface MasterServices extends Server { MasterCoprocessorHost getMasterCoprocessorHost(); /** + * @return Master's coprocessors' coprocessor attributes + */ + Map> getMasterCoprocessorAttributes(); + + /** * @return Master's instance of {@link MasterQuotaManager} */ MasterQuotaManager getMasterQuotaManager(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 76eba61..89e461f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.ClockOutOfSyncException; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.CoordinatedStateManagerFactory; +import org.apache.hadoop.hbase.CoprocessorAttribute; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; @@ -2624,6 +2625,30 @@ public class HRegionServer extends HasThread implements return this.rsHost; } + public Map> getRegionServerCoprocessorAttributes() { + Map> atts = + new HashMap>(); + try { + atts.putAll(getWAL(null).getCoprocessorHost().getCoprocessorAttributes()); + } catch (IOException exception) { + LOG.warn("Exception attempting to fetch wal coprocessor attributes for the common wal; " + + "skipping."); + LOG.debug("Exception details for failure to fetch wal coprocessor attributes.", exception); + } + Collection regions = getOnlineRegionsLocalContext(); + for (Region region : regions) { + atts.putAll(region.getCoprocessorHost().getCoprocessorAttributes()); + try { + atts.putAll(getWAL(region.getRegionInfo()).getCoprocessorHost().getCoprocessorAttributes()); + } catch (IOException exception) { + LOG.warn("Exception attempting to fetch wal coprocessor attributes for region " + region + + "; skipping."); + LOG.debug("Exception details for failure to fetch wal coprocessor attributes.", exception); + } + } + return atts; + } + @Override public ConcurrentMap getRegionsInTransitionInRS() { return this.regionsInTransitionInRS; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorAttributes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorAttributes.java new file mode 100644 index 0000000..30ebd06 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorAttributes.java @@ -0,0 +1,267 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.coprocessor; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.CoprocessorAttribute; +import org.apache.hadoop.hbase.CoprocessorAttributes; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.TestTableLockManager.TestAlterAndDisableMasterObserver; +import org.apache.hadoop.hbase.testclassification.CoprocessorTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * End-to-end test cases to cover from coprocessor attribute instantiation to retrieval from API. + */ +@Category({ CoprocessorTests.class, SmallTests.class }) +public class TestCoprocessorAttributes { + protected static Log myLog = LogFactory.getLog(TestAggregateProtocol.class); + + /** + * Creating the test infrastructure. + */ + private static final TableName TEST_TABLE = TableName.valueOf("TestTable"); + private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily"); + + private static byte[] ROW = Bytes.toBytes("testRow"); + private static final int ROWSIZE = 20; + private static final int rowSeperator1 = 5; + private static final int rowSeperator2 = 12; + private static byte[][] ROWS = makeN(ROW, ROWSIZE); + + private static List attList = new ArrayList(); + + private static HBaseTestingUtility util = new HBaseTestingUtility(); + private static Configuration conf = util.getConfiguration(); + + /** + * A set up method to start the test cluster and register various v is registered and will be + * loaded during region startup. + * @throws Exception + */ + @BeforeClass + public static void setupBeforeClass() throws Exception { + + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + TestCoprocessorAttributes.CoprocessorWithNullAttributes.class.getName() + "," + + TestCoprocessorAttributes.CoprocessorWithAttributes.class.getName()); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + TestCoprocessorAttributes.CoprocessorWithAttributes.class.getName() + "," + + TestAlterAndDisableMasterObserver.class.getName()); + + util.startMiniCluster(1); + final byte[][] SPLIT_KEYS = new byte[][] { ROWS[rowSeperator1], ROWS[rowSeperator2] }; + Table table = util.createTable(TEST_TABLE, TEST_FAMILY, SPLIT_KEYS); + table.close(); + + attList.add(new CoprocessorAttribute("Key 1", "Value 1", "Description 1")); + attList.add(new CoprocessorAttribute("Key 2", "Value 2")); + } + + /** + * Shutting down the cluster + * @throws Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + util.shutdownMiniCluster(); + } + + /** + * an infrastructure method to prepare rows for the testtable. + * @param base + * @param n + * @return + */ + private static byte[][] makeN(byte[] base, int n) { + byte[][] ret = new byte[n][]; + for (int i = 0; i < n; i++) { + ret[i] = Bytes.add(base, Bytes.toBytes(i)); + } + return ret; + } + + /** + * Test's coprocessor is a region coprocessor that implements CoprocessorAttributes but does not + * return any actual attributes. Test verifies that CoprocessorAttributes handles null values. + */ + @Test(timeout = 10000) + public void testNullCoprocessorAttributes() { + String className = TestCoprocessorAttributes.CoprocessorWithNullAttributes.class.getName(); + String classSimpleName = + TestCoprocessorAttributes.CoprocessorWithNullAttributes.class.getSimpleName(); + assertTrue(CoprocessorHost.getLoadedCoprocessors().contains(className)); + + // CoprocessorAttribute should not be at Master level. + Map> mastCoprocAtts = + util.getHBaseCluster().getMaster().getMasterCoprocessorAttributes(); + assertFalse(mastCoprocAtts.containsKey(classSimpleName)); + + // CoprocessorAttribute should be at RS level but an empty list of attributes. + Map> rsCoprocAtts = + util.getHBaseCluster().getRegionServer(0).getRegionServerCoprocessorAttributes(); + assertTrue(rsCoprocAtts.containsKey(classSimpleName)); + assertEquals(new ArrayList(), rsCoprocAtts.get(classSimpleName)); + } + + /** + * Test has region and master coprocessors that implement CoprocessorAttributes. Test verifies + * that CoprocessorAttributes are listed in the correct place for both coprocessors. + */ + @Test(timeout = 10000) + public void testCoprocessorAttributes() { + String classNameWithAtts = TestCoprocessorAttributes.CoprocessorWithAttributes.class.getName(); + String classSimpleNameWithAtts = + TestCoprocessorAttributes.CoprocessorWithAttributes.class.getSimpleName(); + assertTrue(CoprocessorHost.getLoadedCoprocessors().contains(classNameWithAtts)); + + String classNameAltDis = TestAlterAndDisableMasterObserver.class.getName(); + String classSimpleNameAltDis = TestAlterAndDisableMasterObserver.class.getSimpleName(); + assertTrue(CoprocessorHost.getLoadedCoprocessors().contains(classNameAltDis)); + + // TestCoprocessorAttributes and TestAlterAndDisableMO should be at Master level. + Map> mastCoprocAtts = + util.getHBaseCluster().getMaster().getMasterCoprocessorAttributes(); + assertTrue(mastCoprocAtts.containsKey(classSimpleNameWithAtts)); + List atts = mastCoprocAtts.get(classSimpleNameWithAtts); + assertEquals(atts, attList); + assertTrue(mastCoprocAtts.containsKey(classSimpleNameAltDis)); + atts = mastCoprocAtts.get(classSimpleNameAltDis); + assertTrue(atts.size() == 2); + assertEquals("Key 1", atts.get(0).getName()); + assertEquals("Value 1", atts.get(0).getValue()); + assertEquals("Description 1", atts.get(0).getDescription()); + + // TestCoprocessorAttributes but not TestAlterAndDisableMO should be at Region level. + Map> rsCoprocAtts = + util.getHBaseCluster().getRegionServer(0).getRegionServerCoprocessorAttributes(); + assertTrue(rsCoprocAtts.containsKey(classSimpleNameWithAtts)); + atts = rsCoprocAtts.get(classSimpleNameWithAtts); + assertEquals(atts, attList); + assertFalse(rsCoprocAtts.containsKey(classSimpleNameAltDis)); + } + + /** + * Test's coprocessor is not deployed to cluster but does have CoprocessorAttributes. Test + * verifies that only after dynamic coprocessor load do attributes get listed. + */ + @Test(timeout = 10000) + public void testDynamicLoadedHasCoprocessorAttributes() throws Exception { + String className = CoprocessorWithDynamicAttributes.class.getName(); + String classSimpleName = CoprocessorWithDynamicAttributes.class.getSimpleName(); + assertFalse(CoprocessorHost.getLoadedCoprocessors().contains(className)); + + // CoprocessorAttribute should not be at Master level. + Map> mastCoprocAtts = + util.getHBaseCluster().getMaster().getMasterCoprocessorAttributes(); + assertFalse(mastCoprocAtts.containsKey(classSimpleName)); + + // CoprocessorAttribute should not yet be at RS level. + Map> rsCoprocAtts = + util.getHBaseCluster().getRegionServer(0).getRegionServerCoprocessorAttributes(); + assertFalse(rsCoprocAtts.containsKey(classSimpleName)); + + // Create table with coprocessor specified. + String tableName = "testDynamicLoadedHasCoprocessorAttributes"; + byte[] CF = Bytes.toBytes("CF"); + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); + desc.addFamily(new HColumnDescriptor(CF)); + desc.addCoprocessor(className, null, Coprocessor.PRIORITY_USER, null); + Admin adm = util.getAdmin(); + adm.createTable(desc); + + // CoprocessorAttribute should still not be at Master level. + mastCoprocAtts = util.getHBaseCluster().getMaster().getMasterCoprocessorAttributes(); + assertFalse(mastCoprocAtts.containsKey(classSimpleName)); + + // CoprocessorAttribute should now be at RS level. + rsCoprocAtts = util.getHBaseCluster().getRegionServer(0).getRegionServerCoprocessorAttributes(); + assertTrue(rsCoprocAtts.containsKey(classSimpleName)); + List atts = rsCoprocAtts.get(classSimpleName); + assertEquals(atts, attList); + } + + public static class CoprocessorWithNullAttributes implements Coprocessor, CoprocessorAttributes { + @Override + public void start(CoprocessorEnvironment env) throws IOException { + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { + } + + @Override + public List coprocessorAttributes() { + return null; + } + } + + public static class CoprocessorWithAttributes implements Coprocessor, CoprocessorAttributes { + @Override + public void start(CoprocessorEnvironment env) throws IOException { + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { + } + + @Override + public List coprocessorAttributes() { + return attList; + } + } + + public static class CoprocessorWithDynamicAttributes + implements Coprocessor, CoprocessorAttributes { + @Override + public void start(CoprocessorEnvironment env) throws IOException { + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { + } + + @Override + public List coprocessorAttributes() { + return attList; + } + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java index 60b62e4..0d6a272 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java @@ -19,10 +19,12 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; import java.util.List; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; +import org.apache.hadoop.hbase.CoprocessorAttribute; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; @@ -331,4 +333,9 @@ public class MockNoopMasterServices implements MasterServices, Server { public MasterProcedureManagerHost getMasterProcedureManagerHost() { return null; } + + @Override + public Map> getMasterCoprocessorAttributes() { + return null; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java index 02f01c4..047316a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java @@ -22,14 +22,18 @@ import static org.junit.Assert.*; import java.io.IOException; import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CoprocessorAttribute; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; @@ -111,6 +115,14 @@ public class TestMasterStatusServlet { Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper(); Mockito.doReturn(rms).when(master).getRegionServerMetrics(); + // Fake Coprocessor Attributes + Map> attMap = + new HashMap>(); + List attList = new ArrayList(); + attList.add(Mockito.mock(CoprocessorAttribute.class)); + attMap.put("coprocessor", attList); + Mockito.doReturn(attMap).when(master).getMasterCoprocessorAttributes(); + // Mock admin admin = Mockito.mock(Admin.class); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java index 573fdcb..b6707d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Random; import java.util.concurrent.Callable; @@ -37,6 +38,8 @@ import java.util.concurrent.Future; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.CoprocessorAttribute; +import org.apache.hadoop.hbase.CoprocessorAttributes; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; @@ -168,7 +171,8 @@ public class TestTableLockManager { } } - public static class TestAlterAndDisableMasterObserver extends BaseMasterObserver { + public static class TestAlterAndDisableMasterObserver extends BaseMasterObserver + implements CoprocessorAttributes { @Override public void preAddColumnFamilyAction(ObserverContext ctx, TableName tableName, HColumnDescriptor columnFamily) throws IOException { @@ -209,6 +213,14 @@ public class TestTableLockManager { TableName tableName) throws IOException { Threads.sleep(3000); } + + @Override + public List coprocessorAttributes() { + List atts = new ArrayList(); + atts.add(new CoprocessorAttribute("Key 1", "Value 1", "Description 1")); + atts.add(new CoprocessorAttribute("Key 2", "Value 2")); + return atts; + } } @Test(timeout = 600000) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java index a9115f3..37aba35 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java @@ -20,8 +20,12 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.hadoop.hbase.CoprocessorAttribute; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; @@ -108,6 +112,14 @@ public class TestRSStatusServlet { MetricsHBaseServer ms = Mockito.mock(MetricsHBaseServer.class); Mockito.doReturn(new MetricsHBaseServerWrapperStub()).when(ms).getHBaseServerWrapper(); Mockito.doReturn(ms).when(rpcServer).getMetrics(); + + // Fake Coprocessor Attributes + Map> attMap = + new HashMap>(); + List attList = new ArrayList(); + attList.add(Mockito.mock(CoprocessorAttribute.class)); + attMap.put("coprocessor", attList); + Mockito.doReturn(attMap).when(rs).getRegionServerCoprocessorAttributes(); } @Test diff --git a/hbase-server/src/test/resources/META-INF/services/org.apache.hadoop.hbase.CoprocessorAttributes b/hbase-server/src/test/resources/META-INF/services/org.apache.hadoop.hbase.CoprocessorAttributes new file mode 100644 index 0000000..5a1980b --- /dev/null +++ b/hbase-server/src/test/resources/META-INF/services/org.apache.hadoop.hbase.CoprocessorAttributes @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.hadoop.hbase.coprocessor.TestCoprocessorAttributes$CoprocessorWithAttributes +org.apache.hadoop.hbase.coprocessor.TestCoprocessorAttributes$CoprocessorWithDynamicAttributes +org.apache.hadoop.hbase.coprocessor.TestCoprocessorAttributes$CoprocessorWithNullAttributes +org.apache.hadoop.hbase.master.TestTableLockManager$TestAlterAndDisableMasterObserver \ No newline at end of file -- 2.7.4 (Apple Git-66)