From 8670b277052bc222c8dc6a86d93971523cb8e1f4 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Thu, 25 Apr 2013 14:54:02 -0700 Subject: [PATCH] HBASE-8438 Extend bin/hbase to print a "minimal classpath" For tools like pig and hive, blindly appending the full output of `bin/hbase classpath` to their own CLASSPATH is excessive. They already build CLASSPATH entries for hadoop. All they need from us is the delta entries, the dependencies we require w/o hadoop and all of it's transitive deps. This is also a kindness for Windows, where there's a shorter limit on the length of commandline arguments. See also HIVE-2055 for additional discussion. Example usage: $ ./bin/hbase classpath-min \ -e ".*/hadoop/.*" -e ".*/\.m2/.*" \ "$(./bin/hbase classpath)" 2>/dev/null /Users/ndimiduk/repos/hbase/hbase-client/target/classes:/Users/ndimiduk/repos/hbase/hbase-common/target/test-classes:/Users/ndimiduk/repos/hbase/bin/../lib/*.jar:/Users/ndimiduk/repos/hbase/bin/../hbase-server/target:/Users/ndimiduk/repos/hbase/bin/../conf:/Library/Java/JavaVirtualMachines/1.6.0_37-b06-434.jdk/Contents/Home/lib/tools.jar: --- bin/hbase | 3 + .../hadoop/hbase/util/MinimalClasspathTool.java | 112 +++++++++++++++++++++ .../hbase/util/TestMinimalClasspathTool.java | 47 +++++++++ 3 files changed, 162 insertions(+) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/util/MinimalClasspathTool.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMinimalClasspathTool.java diff --git a/bin/hbase b/bin/hbase index 7c1bfb4..781a68a 100755 --- a/bin/hbase +++ b/bin/hbase @@ -88,6 +88,7 @@ if [ $# = 0 ]; then echo "" echo "PACKAGE MANAGEMENT" echo " classpath dump hbase CLASSPATH" + echo " classpath-min dump a minified CLASSPATH" echo " version print the version" echo "" echo " or" @@ -310,6 +311,8 @@ elif [ "$COMMAND" = "zookeeper" ] ; then elif [ "$COMMAND" = "classpath" ] ; then echo $CLASSPATH exit 0 +elif [ "$COMMAND" = "classpath-min" ] ; then + CLASS='org.apache.hadoop.hbase.util.MinimalClasspathTool' elif [ "$COMMAND" = "version" ] ; then CLASS='org.apache.hadoop.hbase.util.VersionInfo' else diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MinimalClasspathTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MinimalClasspathTool.java new file mode 100644 index 0000000..a6de101 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MinimalClasspathTool.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import org.apache.commons.cli.BasicParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; + +@InterfaceAudience.Private +public class MinimalClasspathTool { + + protected static final Log LOG = LogFactory.getLog(MinimalClasspathTool.class); + private static final String CMD_SYNTAX = "MinimalClasspathTool [-e]* CLASSPATH"; + private static final String HELP = + "Minimal Classpath Tool will parse a PATH-like structure and remove" + + " duplicate entries. Two entries are considered duplicates when their" + + " basenames are identical. Optionally, specify a set of regular expression" + + " exclusions to apply to each path entry. Any entries which match any of" + + " the patterns will be omitted from the final PATH. Upon success, the final" + + " path is printed to stdout."; + + /** + * Remove duplicate entries from a PATH-like string. + * @see #HELP + */ + public static String minimize(String cpLike, String[] excludes) { + // watch for null off the parser + if (null == excludes) excludes = new String[0]; + List patterns = new ArrayList(excludes.length); + for (String exclude : excludes) + patterns.add(Pattern.compile(exclude)); + + // build a set of unique entries + String[] entries = cpLike.split(File.pathSeparator); + Map deduped = new HashMap(); + ENTRIES: for (String entry : entries) { + // drop anything that matches an exclude pattern + for (Pattern p : patterns) { + if (p.matcher(entry).matches()) { + LOG.debug("Entry '" + entry + "' matches exclude pattern '" + p + "'. Skipping."); + continue ENTRIES; + } + } + String basename = new File(entry).getName(); + if (!deduped.containsKey(basename)) deduped.put(basename, entry); + } + + StringBuilder sb = new StringBuilder(); + for (Map.Entry e : deduped.entrySet()) { + sb.append(e.getValue()).append(File.pathSeparator); + } + return sb.toString(); + } + + /** + * Produces specialized classpath constructions. + * @see #HELP + */ + public static void main(String[] args) { + Options opts = new Options(); + opts.addOption("h", "help", false, "Print help and exit."); + opts.addOption("e", "exclude", true, + "entries to exclude, matched against the full path. Accepts Java regex patterns."); + CommandLineParser parser = new BasicParser(); + CommandLine cli; + HelpFormatter help = new HelpFormatter(); + try { + cli = parser.parse(opts, args, true); + if (null != cli.getOptionValue("help")) { + help.printHelp(CMD_SYNTAX, HELP, opts, ""); + System.exit(0); + } + if (1 != cli.getArgs().length) { + help.printHelp(CMD_SYNTAX, HELP, opts, ""); + System.exit(1); + } + System.out.println(minimize(cli.getArgs()[0], cli.getOptionValues("exclude"))); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + help.printHelp(CMD_SYNTAX, opts); + System.exit(1); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMinimalClasspathTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMinimalClasspathTool.java new file mode 100644 index 0000000..a4ce65f --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMinimalClasspathTool.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import static org.apache.hadoop.hbase.util.MinimalClasspathTool.minimize; +import static org.junit.Assert.assertEquals; + +import java.io.File; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestMinimalClasspathTool { + + private static final String SEP = File.pathSeparator; + + @Test + public void testDupes() { + assertEquals( + "/foo/bar" + SEP, + minimize("/foo/bar" + SEP + "/foo/bar", null)); + assertEquals( + "/foo/bar" + SEP, + minimize("/foo/bar" + SEP + "/bub/bar", null)); + assertEquals( + "", + minimize("/foo/bar" + SEP + "/bub/bar", new String[] { ".*bar.*" })); + } +} -- 1.8.1