Index: dev-support/generate-hadoopX-poms.sh
===================================================================
--- dev-support/generate-hadoopX-poms.sh (revision 0)
+++ dev-support/generate-hadoopX-poms.sh (working copy)
@@ -0,0 +1,133 @@
+#!/bin/bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+# Generates a pom.xml with a haoop1 or a hadoop2 suffix which can
+# then be used for generating hadoop1 or hadoop2 suitable hbase's
+# fit for publishing to a maven repository. Use these poms also
+# making tarballs to release. The original pom.xml is untouched.
+#
+# This script exists because we cannot figure how to publish
+# into the local (or remote) repositories artifacts that can be
+# used by downstream projects (with poms describing necessary
+# includes). See HBASE-8224 and HBASE-8488 for background.
+#
+# Generation is done by replacing values in original pom and
+# enabling appropriate profile using the '!' trick in the
+# hbase.profile property (this is fragile!) so no need to specify
+# profile on the command line. The original pom.xml should be
+# what we maintain adding in new depdencies, etc., as needed.
+# This script should make it through most any change to the
+# original.
+#
+# Here is how you would build an hbase against hadoop2 and publish
+# the artifacts to your local repo:
+#
+# First run this script passing in current project version and what
+# version you would like the generated artifacts to have. Include
+# either -hadoop1 if built against hadoop1 or -hadoop2 if build against
+# hadoop2. These substrings are expected as part of the new version.
+#
+# $ bash -x ./dev-support/generate-hadoopX-poms.sh 0.95.2-SNAPSHOT 0.95.2-hadoop2-SNAPSHOT
+#
+# This will generate new poms beside current pom.xml made from the
+# origin pom.xml but with a hadoop1 or hadoop2 suffix dependent on
+# what you passed for a new version. Now build passing generated
+# pom name as the pom mvn should use. For example, say we were building
+# hbase against hadoop2:
+#
+#$ mvn clean install -DskipTests -f pom.xml.hadoop2
+#
+
+# TODO: Generate new poms into target dirs so doesn't pollute src tree.
+# It is a little awkward to do since parent pom needs to be able to find
+# the child modules and the child modules need to be able to get to the
+# parent.
+
+function usage {
+ echo "Usage: $0 CURRENT_VERSION NEW_VERSION"
+ echo "For example, $0 hbase-0.95.2-SNAPSHOT hbase-0.95.2-hadoop1-SNAPSHOT"
+ echo "Presumes VERSION has hadoop1 or hadoop2 in it."
+ exit 1
+}
+
+if [[ "$#" -ne 2 ]]; then usage; fi
+old_hbase_version="$1"
+new_hbase_version="$2"
+# Get hadoop version from the new hbase version
+hadoop_version=`echo "$new_hbase_version" | sed -n 's/.*-\(hadoop[12]\)-.*/\1/p'`
+if [[ -z $hadoop_version ]]; then usage ; fi
+
+# Get dir to operate in
+hbase_home="${HBASE_HOME}"
+if [[ -z "$hbase_home" ]]; then
+ here="`dirname \"$0\"`" # relative
+ here="`( cd \"$here\" && pwd )`" # absolutized and normalized
+ if [ -z "$here" ] ; then
+ # error; for some reason, the path is not accessible
+ # to the script (e.g. permissions re-evaled after suid)
+ exit 1 # fail
+ fi
+ hbase_home="`dirname \"$here\"`"
+fi
+
+# Now figure which profile to activate.
+h1=
+h2=
+default='
Table of Contents
@@ -30,94 +30,94 @@
- Constraints are designed to be configurable, so a constraint can be used across different tables, but implement different + Constraints are designed to be configurable, so a constraint can be used across different tables, but implement different behavior depending on the specific configuration given to that constraint.
- By adding a constraint to a table (see Example Usage), constraints will automatically enabled. - You also then have the option of to disable (just 'turn off') or remove (delete all associated information) all constraints on a table. - If you remove all constraints - (see {@link org.apache.hadoop.hbase.constraint.Constraints#remove(org.apache.hadoop.hbase.HTableDescriptor)}, - you must re-add any {@link org.apache.hadoop.hbase.constraint.Constraint} you want on that table. - However, if they are just disabled (see {@link org.apache.hadoop.hbase.constraint.Constraints#disable(org.apache.hadoop.hbase.HTableDescriptor)}, + By adding a constraint to a table (see Example Usage), constraints will automatically enabled. + You also then have the option of to disable (just 'turn off') or remove (delete all associated information) all constraints on a table. + If you remove all constraints + (see {@link org.apache.hadoop.hbase.constraint.Constraints#remove(org.apache.hadoop.hbase.HTableDescriptor)}, + you must re-add any {@link org.apache.hadoop.hbase.constraint.Constraint} you want on that table. + However, if they are just disabled (see {@link org.apache.hadoop.hbase.constraint.Constraints#disable(org.apache.hadoop.hbase.HTableDescriptor)}, all you need to do is enable constraints again, and everything will be turned back on as it was configured. Individual constraints can also be individually enabled, disabled or removed without affecting other constraints.
- By default, constraints are disabled on a table. + By default, constraints are disabled on a table. This means you will not see any slow down on a table if constraints are not enabled.
- Locking is recommended around each of Constraints add methods: - {@link org.apache.hadoop.hbase.constraint.Constraints#add(org.apache.hadoop.hbase.HTableDescriptor, Class...)}, - {@link org.apache.hadoop.hbase.constraint.Constraints#add(org.apache.hadoop.hbase.HTableDescriptor, org.apache.hadoop.hbase.util.Pair...)}, + Locking is recommended around each of Constraints add methods: + {@link org.apache.hadoop.hbase.constraint.Constraints#add(org.apache.hadoop.hbase.HTableDescriptor, Class...)}, + {@link org.apache.hadoop.hbase.constraint.Constraints#add(org.apache.hadoop.hbase.HTableDescriptor, org.apache.hadoop.hbase.util.Pair...)}, and {@link org.apache.hadoop.hbase.constraint.Constraints#add(org.apache.hadoop.hbase.HTableDescriptor, Class, org.apache.hadoop.conf.Configuration)}. Any changes on a single HTableDescriptor should be serialized, either within a single thread or via external mechanisms.
- Note that having a higher priority means that a constraint will run later; e.g. a constraint with priority 1 will run before a - constraint with priority 2. + Note that having a higher priority means that a constraint will run later; e.g. a constraint with priority 1 will run before a + constraint with priority 2.
- Since Constraints currently are designed to just implement simple checks (e.g. is the value in the right range), there will - be no atomicity conflicts. - Even if one of the puts finishes the constraint first, the single row will not be corrupted and the 'fastest' write will win; + Since Constraints currently are designed to just implement simple checks (e.g. is the value in the right range), there will + be no atomicity conflicts. + Even if one of the puts finishes the constraint first, the single row will not be corrupted and the 'fastest' write will win; the underlying region takes care of breaking the tie and ensuring that writes get serialized to the table. - So yes, this doesn't ensure that we are going to get specific ordering or even a fully consistent view of the underlying data. + So yes, this doesn't ensure that we are going to get specific ordering or even a fully consistent view of the underlying data.
Each constraint should only use local/instance variables, unless doing more advanced usage. Static variables could cause difficulties when checking concurrent writes to the same region, leading to either highly locked situations (decreasing through-put) or higher probability of errors. However, as long as each constraint just uses local variables, each thread interacting with the constraint will execute correctly and efficiently.
- Under the hood, constraints are implemented as a Coprocessor (see {@link org.apache.hadoop.hbase.constraint.ConstraintProcessor} +
+ Under the hood, constraints are implemented as a Coprocessor (see {@link org.apache.hadoop.hbase.constraint.ConstraintProcessor} if you are interested).
- Let's look at one possible implementation of a constraint - an IntegerConstraint(there are also several simple examples in the tests).
+ Let's look at one possible implementation of a constraint - an IntegerConstraint(there are also several simple examples in the tests).
The IntegerConstraint checks to make sure that the value is a String-encoded int.
It is really simple to implement this kind of constraint, the only method needs to be implemented is
{@link org.apache.hadoop.hbase.constraint.Constraint#check(org.apache.hadoop.hbase.client.Put)}:
@@ -141,18 +141,18 @@
} catch (NumberFormatException e) {
throw new ConstraintException("Value in Put (" + p
+ ") was not a String-encoded integer", e);
- } } }
+ } } }
- Note that all exceptions that you expect to be thrown must be caught and then rethrown as a - {@link org.apache.hadoop.hbase.exceptions.ConstraintException}. This way, you can be sure that a - {@link org.apache.hadoop.hbase.client.Put} fails for an expected reason, rather than for any reason. - For example, an {@link java.lang.OutOfMemoryError} is probably indicative of an inherent problem in + Note that all exceptions that you expect to be thrown must be caught and then rethrown as a + {@link org.apache.hadoop.hbase.constraint.ConstraintException}. This way, you can be sure that a + {@link org.apache.hadoop.hbase.client.Put} fails for an expected reason, rather than for any reason. + For example, an {@link java.lang.OutOfMemoryError} is probably indicative of an inherent problem in the {@link org.apache.hadoop.hbase.constraint.Constraint}, rather than a failed {@link org.apache.hadoop.hbase.client.Put}.
If an unexpected exception is thrown (for example, any kind of uncaught {@link java.lang.RuntimeException}), - constraint-checking will be 'unloaded' from the regionserver where that error occurred. + constraint-checking will be 'unloaded' from the regionserver where that error occurred. This means no further {@link org.apache.hadoop.hbase.constraint.Constraint Constraints} will be checked on that server until it is reloaded. This is done to ensure the system remains as available as possible. Therefore, be careful when writing your own Constraint. @@ -166,14 +166,14 @@ Constraints.add(desc, IntegerConstraint.class);
- Once we added the IntegerConstraint, constraints will be enabled on the table (once it is created) and + Once we added the IntegerConstraint, constraints will be enabled on the table (once it is created) and we will always check to make sure that the value is an String-encoded integer. -
+
However, suppose we also write our own constraint, MyConstraint.java.
- First, you need to make sure this class-files are in the classpath (in a jar) on the regionserver where
+ First, you need to make sure this class-files are in the classpath (in a jar) on the regionserver where
that constraint will be run (this could require a rolling restart on the region server - see Caveats above)
- Suppose that MyConstraint also uses a Configuration (see {@link org.apache.hadoop.hbase.constraint.Constraint#getConf()}). + Suppose that MyConstraint also uses a Configuration (see {@link org.apache.hadoop.hbase.constraint.Constraint#getConf()}). Then adding MyConstraint looks like this:
Suppose we realize that the {@link org.apache.hadoop.conf.Configuration} for MyConstraint is actually wrong - when it was added to the table. Note, when it is added to the table, it is not added by reference, + when it was added to the table. Note, when it is added to the table, it is not added by reference, but is instead copied into the {@link org.apache.hadoop.hbase.HTableDescriptor}. Thus, to change the {@link org.apache.hadoop.conf.Configuration} we are using for MyConstraint, we need to do this: @@ -202,7 +202,7 @@ Constraints.setConfiguration(desc, MyConstraint.class, conf);
- This will overwrite the previous configuration for MyConstraint, but not change the order of the + This will overwrite the previous configuration for MyConstraint, but not change the order of the constraint nor if it is enabled/disabled.
Note that the same constraint class can be added multiple times to a table without repercussion. @@ -216,7 +216,7 @@
This just turns off MyConstraint, but retains the position and the configuration associated with MyConstraint. - Now, if we want to re-enable the constraint, its just another one-liner: + Now, if we want to re-enable the constraint, its just another one-liner:
Constraints.enable(desc, MyConstraint.class);
Index: hbase-server/src/main/java/org/apache/hadoop/hbase/thrift/HThreadedSelectorServerArgs.java
===================================================================
--- hbase-server/src/main/java/org/apache/hadoop/hbase/thrift/HThreadedSelectorServerArgs.java (revision 1509460)
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/thrift/HThreadedSelectorServerArgs.java (working copy)
@@ -19,22 +19,20 @@
package org.apache.hadoop.hbase.thrift;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.thrift.server.TThreadedSelectorServer;
import org.apache.thrift.transport.TNonblockingServerTransport;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* A TThreadedSelectorServer.Args that reads hadoop configuration
*/
@InterfaceAudience.Private
public class HThreadedSelectorServerArgs extends TThreadedSelectorServer.Args {
+ private static final Log LOG = LogFactory.getLog(TThreadedSelectorServer.class);
- private static final Logger LOG =
- LoggerFactory.getLogger(TThreadedSelectorServer.class);
-
/**
* Number of selector threads for reading and writing socket
*/
Index: hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
===================================================================
--- hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java (revision 1509460)
+++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java (working copy)
@@ -65,8 +65,8 @@
import org.junit.Test;
import org.mockito.Mockito;
import org.junit.experimental.categories.Category;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
/**
* Standup the master and fake it to test various aspects of master function.
@@ -78,7 +78,7 @@
*/
@Category(MediumTests.class)
public class TestMasterNoCluster {
- private static Logger LOG = LoggerFactory.getLogger(TestMasterNoCluster.class);
+ private static final Log LOG = LogFactory.getLog(TestMasterNoCluster.class);
private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
@BeforeClass
@@ -240,7 +240,7 @@
* @throws IOException
* @throws KeeperException
* @throws InterruptedException
- * @throws DeserializationException
+ * @throws DeserializationException
* @throws ServiceException
*/
@Test (timeout=30000)
Index: pom.xml
===================================================================
--- pom.xml (revision 1509460)
+++ pom.xml (working copy)
@@ -461,6 +461,7 @@
org.apache.maven.plugins
maven-release-plugin
+ 2.4.1
-Dmaven.test.skip.exec
+ pom.xml
@@ -609,6 +611,9 @@
prepare-package
+
test-jar
@@ -884,7 +889,9 @@
2.4
2.6
1.1.1
- 2.1
+ 2.2
+ 3.2.1
+ 3.0.1
2.1.2
12.0.1
1.8.8
@@ -896,13 +903,13 @@
1.6.8
4.11
1.50
- 1.4.3
1.2.17
1.9.0
2.4.1
1.0.1
0.9.0
3.4.5
+ 1.6.4
0.0.1-SNAPSHOT
2.6.3
2.3.1
@@ -1046,6 +1053,18 @@
${jettison.version}
+ log4j
+ log4j
+ ${log4j.version}
+
+
+
+ org.slf4j
+ slf4j-api
+ ${slf4j.version}
+
+
com.yammer.metrics
metrics-core
${metrics-core.version}
@@ -1056,6 +1075,16 @@
${guava.version}
+ commons-collections
+ commons-collections
+ ${collections.version}
+
+
+ commons-httpclient
+ commons-httpclient
+ ${httpclient.version}
+
+
commons-cli
commons-cli
${commons-cli.version}
@@ -1091,11 +1120,6 @@
${commons-math.version}
- log4j
- log4j
- ${log4j.version}
-
-
org.apache.zookeeper
zookeeper
${zookeeper.version}
@@ -1204,16 +1228,6 @@
${jackson.version}
- org.slf4j
- slf4j-api
- ${slf4j.version}
-
-
- org.slf4j
- slf4j-log4j12
- ${slf4j.version}
-
-
junit
@@ -1438,7 +1462,8 @@
hadoop-1.1
- !hadoop.profile
+
+ !hadoop.profile
@@ -1446,7 +1471,6 @@
${hadoop-one.version}
- 1.4.3
hbase-hadoop1-compat
src/main/assembly/hadoop-one-compat.xml
@@ -1507,7 +1531,6 @@
1.0.4
${hadoop.version}
- 1.4.3
hbase-hadoop1-compat
src/main/assembly/hadoop-one-compat.xml
@@ -1558,8 +1581,8 @@
hadoop-2.0
- hadoop.profile
- 2.0
+
+ hadoop.profile 2.0
@@ -1567,7 +1590,6 @@
${hadoop-two.version}
- 1.6.1
hbase-hadoop2-compat
src/main/assembly/hadoop-two-compat.xml
@@ -1575,6 +1597,38 @@
org.apache.hadoop
+ hadoop-mapreduce-client-core
+ ${hadoop-two.version}
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-jobclient
+ ${hadoop-two.version}
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-jobclient
+ ${hadoop-two.version}
+ test-jar
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ ${hadoop-two.version}
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ ${hadoop-two.version}
+ test-jar
+
+
+ org.apache.hadoop
+ hadoop-auth
+ ${hadoop-two.version}
+
+
+ org.apache.hadoop
hadoop-common
${hadoop-two.version}
@@ -1625,7 +1679,6 @@
- 1.6.1
3.0.0-SNAPSHOT