From 16d04e3951a76513d76711c070e3abd9c8839f00 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Fri, 15 Nov 2013 13:22:54 -0800 Subject: [PATCH] HBASE-8438 Extend bin/hbase to print a "mapreduce classpath" $ ./bin/hbase mapredcp --help Usage: hbase mapredcp [-Dtmpjars=...] Construct a CLASSPATH containing dependency jars required to run a mapreduce job. By default, includes any jars detected by TableMapReduceUtils. Provide additional entries by specifying a comma-separated list in tmpjars. $ ./bin/hbase mapredcp | tr ':' '\n' /private/tmp/hbase-0.94.14-SNAPSHOT/hbase-0.94.14-SNAPSHOT.jar /private/tmp/hbase-0.94.14-SNAPSHOT/lib/protobuf-java-2.4.0a.jar /private/tmp/hbase-0.94.14-SNAPSHOT/lib/zookeeper-3.4.5.jar /private/tmp/hbase-0.94.14-SNAPSHOT/lib/guava-11.0.2.jar /private/tmp/hbase-0.94.14-SNAPSHOT/lib/hadoop-core-1.0.4.jar --- bin/hbase | 4 +- .../hadoop/hbase/mapreduce/TableMapReduceUtil.java | 24 ++++++++ .../util/MapreduceDependencyClasspathTool.java | 72 ++++++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/apache/hadoop/hbase/util/MapreduceDependencyClasspathTool.java diff --git a/bin/hbase b/bin/hbase index 841c68c..adf4dae 100755 --- a/bin/hbase +++ b/bin/hbase @@ -94,6 +94,7 @@ if [ $# = 0 ]; then echo "" echo "PACKAGE MANAGEMENT" echo " classpath dump hbase CLASSPATH" + echo " mapredcp dump CLASSPATH entries required by mapreduce" echo " version print the version" echo "" echo " or" @@ -325,7 +326,8 @@ elif [ "$COMMAND" = "zookeeper" ] ; then if [ "$1" != "stop" ] ; then HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS" fi - +elif [ "$COMMAND" = "mapredcp" ] ; then + CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool' elif [ "$COMMAND" = "classpath" ] ; then echo $CLASSPATH exit 0 diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index df1f160..2014e2b 100644 --- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -23,6 +23,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -572,6 +573,29 @@ public static void initCredentials(Job job) throws IOException { org.apache.hadoop.hbase.util.Bytes.class); //one class from hbase.jar } + /** + * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}. + * Also exposed to shell scripts via `bin/hbase mapredcp`. + */ + public static String buildDependencyClasspath(Configuration conf) { + if (conf == null) { + throw new IllegalArgumentException("Must provide a configuration object."); + } + Set paths = new HashSet(conf.getStringCollection("tmpjars")); + if (paths.size() == 0) { + throw new IllegalArgumentException("Configuration contains no tmpjars."); + } + StringBuilder sb = new StringBuilder(); + for (String s : paths) { + // entries can take the form 'file:/path/to/file.jar'. + int idx = s.indexOf(":"); + if (idx != -1) s = s.substring(idx + 1); + if (sb.length() > 0) sb.append(File.pathSeparator); + sb.append(s); + } + return sb.toString(); + } + /** * Add the HBase dependency jars as well as jars for any of the configured * job classes to the job configuration, so that JobClient will ship them diff --git a/src/main/java/org/apache/hadoop/hbase/util/MapreduceDependencyClasspathTool.java b/src/main/java/org/apache/hadoop/hbase/util/MapreduceDependencyClasspathTool.java new file mode 100644 index 0000000..171b34b --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/util/MapreduceDependencyClasspathTool.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * Generate a classpath string containing any jars required by mapreduce jobs. Specify + * additional values by providing a comma-separated list of paths via -Dtmpjars. + */ +public class MapreduceDependencyClasspathTool implements Tool { + + private Configuration conf; + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public int run(String[] args) throws Exception { + if (args.length > 0) { + System.err.println("Usage: hbase mapredcp [-Dtmpjars=...]"); + System.err.println(" Construct a CLASSPATH containing dependency jars required to run a mapreduce"); + System.err.println(" job. By default, includes any jars detected by TableMapReduceUtils. Provide"); + System.err.println(" additional entries by specifying a comma-separated list in tmpjars."); + return 0; + } + + Job job = new Job(getConf()); + TableMapReduceUtil.addDependencyJars(job); + System.out.println(TableMapReduceUtil.buildDependencyClasspath(job.getConfiguration())); + return 0; + } + + public static void main(String[] argv) throws Exception { + // Silence the usual noise. This is probably fragile... + Logger logger = Logger.getLogger("org.apache.hadoop.hbase"); + if (logger != null) { + logger.setLevel(Level.WARN); + } + System.exit(ToolRunner.run( + HBaseConfiguration.create(), new MapreduceDependencyClasspathTool(), argv)); + } +} -- 1.8.4.2