diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 4c7a666..4a4fbba 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -40,6 +40,7 @@ import java.io.StringWriter; import java.lang.RuntimeException; import java.net.URL; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -85,6 +86,10 @@ import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.processors.CommandProcessor; +import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; +import org.apache.hadoop.hive.ql.processors.HiveCommand; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; @@ -94,10 +99,6 @@ import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; -import org.apache.hadoop.hive.ql.processors.EncryptionProcessor; -import org.apache.hadoop.hive.ql.processors.CommandProcessor; -import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.processors.HiveCommand; import com.google.common.collect.ImmutableList; @@ -117,6 +118,7 @@ private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; private final String defaultInitScript = "q_test_init.sql"; private final String defaultCleanupScript = "q_test_cleanup.sql"; + private final String[] testOnlyCommands = new String[]{"crypto"}; private String testWarehouse; private final String testFiles; @@ -966,7 +968,9 @@ private int executeTestCommand(final String command) { String wareHouseDir = SessionState.get().getConf().getVar(ConfVars.METASTOREWAREHOUSE) .replaceAll("^[a-zA-Z]+://.*?:\\d+", ""); commandArgs = commandArgs.replaceAll("\\$\\{hiveconf:hive\\.metastore\\.warehouse\\.dir\\}", - wareHouseDir); + wareHouseDir); + + enableTestOnlyCmd(SessionState.get().getConf()); try { CommandProcessor proc = getTestCommand(commandName); @@ -987,22 +991,25 @@ private int executeTestCommand(final String command) { } } - private CommandProcessor getTestCommand(final String commandName) { + private CommandProcessor getTestCommand(final String commandName) throws SQLException { HiveCommand testCommand = HiveCommand.find(new String[]{commandName}, HiveCommand.ONLY_FOR_TESTING); + if (testCommand == null) { return null; } - switch (testCommand) { - case CRYPTO: - if (hes == null) { - throw new RuntimeException("HDFS encryption is not initialized for testing."); - } + return CommandProcessorFactory + .getForHiveCommandInternal(new String[]{commandName}, SessionState.get().getConf(), + testCommand.isOnlyForTesting()); + } - return new EncryptionProcessor(hes, conf); - default: - throw new IllegalArgumentException("Unknown test command: " + commandName); + private void enableTestOnlyCmd(HiveConf conf){ + StringBuilder securityCMDs = new StringBuilder(conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST)); + for(String c : testOnlyCommands){ + securityCMDs.append(","); + securityCMDs.append(c); } + conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), securityCMDs.toString()); } private boolean isCommandUsedForTesting(final String command) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java index 727f61f..2be6f26 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.session.SessionState; /** @@ -49,8 +50,14 @@ public static CommandProcessor get(String cmd) } public static CommandProcessor getForHiveCommand(String[] cmd, HiveConf conf) - throws SQLException { - HiveCommand hiveCommand = HiveCommand.find(cmd); + throws SQLException { + return getForHiveCommandInternal(cmd, conf, false); + } + + public static CommandProcessor getForHiveCommandInternal(String[] cmd, HiveConf conf, + boolean testOnly) + throws SQLException { + HiveCommand hiveCommand = HiveCommand.find(cmd, testOnly); if (hiveCommand == null || isBlank(cmd[0])) { return null; } @@ -58,7 +65,8 @@ public static CommandProcessor getForHiveCommand(String[] cmd, HiveConf conf) conf = new HiveConf(); } Set availableCommands = new HashSet(); - for (String availableCommand : conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST).split(",")) { + for (String availableCommand : conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST) + .split(",")) { availableCommands.add(availableCommand.toLowerCase().trim()); } if (!availableCommands.contains(cmd[0].trim().toLowerCase())) { @@ -82,6 +90,12 @@ public static CommandProcessor getForHiveCommand(String[] cmd, HiveConf conf) return new CompileProcessor(); case RELOAD: return new ReloadProcessor(); + case CRYPTO: + try { + return new CryptoProcessor(SessionState.get().getHdfsEncryptionShim(), conf); + } catch (HiveException e) { + throw new SQLException("Fail to start the command processor due to the exception: ", e); + } default: throw new AssertionError("Unknown HiveCommand " + hiveCommand); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java new file mode 100644 index 0000000..5eaadbb --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CryptoProcessor.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.processors; + +import org.apache.commons.cli.Options; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.CommandNeedRetryException; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.HadoopShims; + +import java.io.IOException; +import java.util.Arrays; + +/** + * This class processes HADOOP commands used for HDFS encryption. It is meant to be run + * only by Hive unit & queries tests. + */ +public class CryptoProcessor implements CommandProcessor { + public static final Log LOG = LogFactory.getLog(CryptoProcessor.class.getName()); + + private HadoopShims.HdfsEncryptionShim encryptionShim; + + private Options CREATE_KEY_OPTIONS; + private Options DELETE_KEY_OPTIONS; + private Options CREATE_ZONE_OPTIONS; + + private int DEFAULT_BIT_LENGTH = 128; + + private HiveConf conf; + + public CryptoProcessor(HadoopShims.HdfsEncryptionShim encryptionShim, HiveConf conf) { + this.encryptionShim = encryptionShim; + this.conf = conf; + + CREATE_KEY_OPTIONS = new Options(); + CREATE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); + CREATE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("bitLength").create()); // optional + + DELETE_KEY_OPTIONS = new Options(); + DELETE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); + + CREATE_ZONE_OPTIONS = new Options(); + CREATE_ZONE_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); + CREATE_ZONE_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("path").isRequired().create()); + } + + private CommandLine parseCommandArgs(final Options opts, String[] args) throws ParseException { + CommandLineParser parser = new GnuParser(); + return parser.parse(opts, args); + } + + private CommandProcessorResponse returnErrorResponse(final String errmsg) { + return new CommandProcessorResponse(1, "Encryption Processor Helper Failed:" + errmsg, null); + } + + private void writeTestOutput(final String msg) { + SessionState.get().out.println(msg); + } + + @Override + public void init() { + } + + @Override + public CommandProcessorResponse run(String command) throws CommandNeedRetryException { + String[] args = command.split("\\s+"); + + if (args.length < 1) { + return returnErrorResponse("Command arguments are empty."); + } + + if (encryptionShim == null) { + return returnErrorResponse("Hadoop encryption shim is not initialized."); + } + + String action = args[0]; + String params[] = Arrays.copyOfRange(args, 1, args.length); + + try { + if (action.equalsIgnoreCase("create_key")) { + createEncryptionKey(params); + } else if (action.equalsIgnoreCase("create_zone")) { + createEncryptionZone(params); + } else if (action.equalsIgnoreCase("delete_key")) { + deleteEncryptionKey(params); + } else { + return returnErrorResponse("Unknown command action: " + action); + } + } catch (Exception e) { + return returnErrorResponse(e.getMessage()); + } + + return new CommandProcessorResponse(0); + } + + /** + * Creates an encryption key using the parameters passed through the 'create_key' action. + * + * @param params Parameters passed to the 'create_key' command action. + * @throws Exception If key creation failed. + */ + private void createEncryptionKey(String[] params) throws Exception { + CommandLine args = parseCommandArgs(CREATE_KEY_OPTIONS, params); + + String keyName = args.getOptionValue("keyName"); + String bitLength = args.getOptionValue("bitLength", Integer.toString(DEFAULT_BIT_LENGTH)); + + try { + encryptionShim.createKey(keyName, new Integer(bitLength)); + } catch (Exception e) { + throw new Exception("Cannot create encryption key: " + e.getMessage()); + } + + writeTestOutput("Encryption key created: '" + keyName + "'"); + } + + /** + * Creates an encryption zone using the parameters passed through the 'create_zone' action. + * + * @param params Parameters passed to the 'create_zone' command action. + * @throws Exception If zone creation failed. + */ + private void createEncryptionZone(String[] params) throws Exception { + CommandLine args = parseCommandArgs(CREATE_ZONE_OPTIONS, params); + + String keyName = args.getOptionValue("keyName"); + Path cryptoZone = new Path(args.getOptionValue("path")); + if (cryptoZone == null) { + throw new Exception("Cannot create encryption zone: Invalid path '" + + args.getOptionValue("path") + "'"); + } + + try { + encryptionShim.createEncryptionZone(cryptoZone, keyName); + } catch (IOException e) { + throw new Exception("Cannot create encryption zone: " + e.getMessage()); + } + + writeTestOutput("Encryption zone created: '" + cryptoZone + "' using key: '" + keyName + "'"); + } + + /** + * Deletes an encryption key using the parameters passed through the 'delete_key' action. + * + * @param params Parameters passed to the 'delete_key' command action. + * @throws Exception If key deletion failed. + */ + private void deleteEncryptionKey(String[] params) throws Exception { + CommandLine args = parseCommandArgs(DELETE_KEY_OPTIONS, params); + + String keyName = args.getOptionValue("keyName"); + try { + encryptionShim.deleteKey(keyName); + } catch (IOException e) { + throw new Exception("Cannot delete encryption key: " + e.getMessage()); + } + + writeTestOutput("Encryption key deleted: '" + keyName + "'"); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/EncryptionProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/EncryptionProcessor.java deleted file mode 100644 index e1ec61c..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/EncryptionProcessor.java +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.processors; - -import org.apache.commons.cli.Options; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.shims.HadoopShims; - -import java.io.IOException; -import java.util.Arrays; - -/** - * This class processes HADOOP commands used for HDFS encryption. It is meant to be run - * only by Hive unit & queries tests. - */ -public class EncryptionProcessor implements CommandProcessor { - public static final Log LOG = LogFactory.getLog(EncryptionProcessor.class.getName()); - - private HadoopShims.HdfsEncryptionShim encryptionShim; - - private Options CREATE_KEY_OPTIONS; - private Options DELETE_KEY_OPTIONS; - private Options CREATE_ZONE_OPTIONS; - - private int DEFAULT_BIT_LENGTH = 128; - - private HiveConf conf; - - public EncryptionProcessor(HadoopShims.HdfsEncryptionShim encryptionShim, HiveConf conf) { - this.encryptionShim = encryptionShim; - this.conf = conf; - - CREATE_KEY_OPTIONS = new Options(); - CREATE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); - CREATE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("bitLength").create()); // optional - - DELETE_KEY_OPTIONS = new Options(); - DELETE_KEY_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); - - CREATE_ZONE_OPTIONS = new Options(); - CREATE_ZONE_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("keyName").isRequired().create()); - CREATE_ZONE_OPTIONS.addOption(OptionBuilder.hasArg().withLongOpt("path").isRequired().create()); - } - - private CommandLine parseCommandArgs(final Options opts, String[] args) throws ParseException { - CommandLineParser parser = new GnuParser(); - return parser.parse(opts, args); - } - - private CommandProcessorResponse returnErrorResponse(final String errmsg) { - return new CommandProcessorResponse(1, "Encryption Processor Helper Failed:" + errmsg, null); - } - - private void writeTestOutput(final String msg) { - SessionState.get().out.println(msg); - } - - @Override - public void init() { - } - - @Override - public CommandProcessorResponse run(String command) throws CommandNeedRetryException { - String[] args = command.split("\\s+"); - - if (args.length < 1) { - return returnErrorResponse("Command arguments are empty."); - } - - if (encryptionShim == null) { - return returnErrorResponse("Hadoop encryption shim is not initialized."); - } - - String action = args[0]; - String params[] = Arrays.copyOfRange(args, 1, args.length); - - try { - if (action.equalsIgnoreCase("create_key")) { - createEncryptionKey(params); - } else if (action.equalsIgnoreCase("create_zone")) { - createEncryptionZone(params); - } else if (action.equalsIgnoreCase("delete_key")) { - deleteEncryptionKey(params); - } else { - return returnErrorResponse("Unknown command action: " + action); - } - } catch (Exception e) { - return returnErrorResponse(e.getMessage()); - } - - return new CommandProcessorResponse(0); - } - - /** - * Creates an encryption key using the parameters passed through the 'create_key' action. - * - * @param params Parameters passed to the 'create_key' command action. - * @throws Exception If key creation failed. - */ - private void createEncryptionKey(String[] params) throws Exception { - CommandLine args = parseCommandArgs(CREATE_KEY_OPTIONS, params); - - String keyName = args.getOptionValue("keyName"); - String bitLength = args.getOptionValue("bitLength", Integer.toString(DEFAULT_BIT_LENGTH)); - - try { - encryptionShim.createKey(keyName, new Integer(bitLength)); - } catch (Exception e) { - throw new Exception("Cannot create encryption key: " + e.getMessage()); - } - - writeTestOutput("Encryption key created: '" + keyName + "'"); - } - - /** - * Creates an encryption zone using the parameters passed through the 'create_zone' action. - * - * @param params Parameters passed to the 'create_zone' command action. - * @throws Exception If zone creation failed. - */ - private void createEncryptionZone(String[] params) throws Exception { - CommandLine args = parseCommandArgs(CREATE_ZONE_OPTIONS, params); - - String keyName = args.getOptionValue("keyName"); - Path cryptoZone = new Path(args.getOptionValue("path")); - if (cryptoZone == null) { - throw new Exception("Cannot create encryption zone: Invalid path '" - + args.getOptionValue("path") + "'"); - } - - try { - encryptionShim.createEncryptionZone(cryptoZone, keyName); - } catch (IOException e) { - throw new Exception("Cannot create encryption zone: " + e.getMessage()); - } - - writeTestOutput("Encryption zone created: '" + cryptoZone + "' using key: '" + keyName + "'"); - } - - /** - * Deletes an encryption key using the parameters passed through the 'delete_key' action. - * - * @param params Parameters passed to the 'delete_key' command action. - * @throws Exception If key deletion failed. - */ - private void deleteEncryptionKey(String[] params) throws Exception { - CommandLine args = parseCommandArgs(DELETE_KEY_OPTIONS, params); - - String keyName = args.getOptionValue("keyName"); - try { - encryptionShim.deleteKey(keyName); - } catch (IOException e) { - throw new Exception("Cannot delete encryption key: " + e.getMessage()); - } - - writeTestOutput("Encryption key deleted: '" + keyName + "'"); - } -} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java index ac5053a..21bdcf4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java @@ -20,15 +20,17 @@ import java.sql.SQLException; -import junit.framework.Assert; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; public class TestCommandProcessorFactory { + private final String[] testOnlyCommands = new String[]{"crypto"}; + private HiveConf conf; @Before @@ -38,27 +40,39 @@ public void setUp() throws Exception { @Test public void testInvalidCommands() throws Exception { - Assert.assertNull("Null should have returned null", CommandProcessorFactory.getForHiveCommand(null, conf)); - Assert.assertNull("Blank should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{" "}, conf)); - Assert.assertNull("set role should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"set role"}, conf)); - Assert.assertNull("SQL should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"SELECT * FROM TABLE"}, conf)); + Assert.assertNull("Null should have returned null", + CommandProcessorFactory.getForHiveCommand(null, conf)); + Assert.assertNull("Blank should have returned null", + CommandProcessorFactory.getForHiveCommand(new String[]{" "}, conf)); + Assert.assertNull("Set role should have returned null", + CommandProcessorFactory.getForHiveCommand(new String[]{"set role"}, conf)); + Assert.assertNull("SQL should have returned null", + CommandProcessorFactory.getForHiveCommand(new String[]{"SELECT * FROM TABLE"}, conf)); + Assert.assertNull("Test only command should have returned null", + CommandProcessorFactory.getForHiveCommand(new String[]{"CRYPTO --listZones"}, conf)); } + @Test public void testAvailableCommands() throws Exception { + enableTestOnlyCmd(conf); SessionState.start(conf); + for (HiveCommand command : HiveCommand.values()) { String cmd = command.name(); - Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf)); - } - for (HiveCommand command : HiveCommand.values()) { - String cmd = command.name().toLowerCase(); - Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf)); + String cmdInLowerCase = cmd.toLowerCase(); + Assert.assertNotNull("Cmd " + cmd + " not return null", + CommandProcessorFactory + .getForHiveCommandInternal(new String[]{cmd}, conf, command.isOnlyForTesting())); + Assert.assertNotNull("Cmd " + cmd + " not return null", + CommandProcessorFactory.getForHiveCommandInternal( + new String[]{cmdInLowerCase}, conf, command.isOnlyForTesting())); } conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), ""); for (HiveCommand command : HiveCommand.values()) { String cmd = command.name(); try { - CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf); + CommandProcessorFactory + .getForHiveCommandInternal(new String[]{cmd}, conf, command.isOnlyForTesting()); Assert.fail("Expected SQLException for " + cmd + " as available commands is empty"); } catch (SQLException e) { Assert.assertEquals("Insufficient privileges to execute " + cmd, e.getMessage()); @@ -66,4 +80,13 @@ public void testAvailableCommands() throws Exception { } } } + + private void enableTestOnlyCmd(HiveConf conf){ + StringBuilder securityCMDs = new StringBuilder(conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST)); + for(String c : testOnlyCommands){ + securityCMDs.append(","); + securityCMDs.append(c); + } + conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), securityCMDs.toString()); + } }