diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLOperationsOnEncryptedZones.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLOperationsOnEncryptedZones.java new file mode 100644 index 0000000000..fa749abfee --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLOperationsOnEncryptedZones.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.TestReplicationScenarios; +import org.apache.hadoop.hive.ql.parse.WarehouseInstance; +import org.apache.hadoop.hive.shims.Utils; +import org.junit.BeforeClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Test; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; + +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_ENABLED; +import static org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION; + +public class TestDDLOperationsOnEncryptedZones { + private static String jksFile = System.getProperty("java.io.tmpdir") + "/test.jks"; + @Rule + public final TestName testName = new TestName(); + + protected static final Logger LOG = LoggerFactory.getLogger(TestReplicationScenarios.class); + private static WarehouseInstance primary; + private static String primaryDbName; + private static Configuration conf; + private static MiniDFSCluster miniDFSCluster; + + @BeforeClass + public static void beforeClassSetup() throws Exception { + conf = new Configuration(); + conf.set("dfs.client.use.datanode.hostname", "true"); + conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); + conf.set("hadoop.security.key.provider.path", "jceks://file" + jksFile); + conf.setBoolean("dfs.namenode.delegation.token.always-use", true); + + conf.setLong(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE.varname, 1); + conf.setLong(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXNUMFILES.varname, 0); + conf.setBoolean(METASTORE_AGGREGATE_STATS_CACHE_ENABLED.varname, false); + + miniDFSCluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); + + DFSTestUtil.createKey("test_key", miniDFSCluster, conf); + primary = new WarehouseInstance(LOG, miniDFSCluster, new HashMap() {{ + put(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false"); + put(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname, "true"); + put("hive.repl.bootstrap.dump.open.txn.timeout", "1s"); + put("hive.strict.checks.bucketing", "false"); + put("hive.mapred.mode", "nonstrict"); + put("mapred.input.dir.recursive", "true"); + put("hive.metastore.disallow.incompatible.col.type.changes", "false"); + put("hive.strict.managed.tables", "false"); + + put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(), + GzipJSONMessageEncoder.class.getCanonicalName()); + put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true"); + put(HiveConf.ConfVars.HIVE_TXN_MANAGER.varname, + "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); + put(MetastoreConf.ConfVars.CAPABILITY_CHECK.getHiveName(), "false"); + put(HiveConf.ConfVars.REPL_BOOTSTRAP_DUMP_OPEN_TXN_TIMEOUT.varname, "1s"); + put(HiveConf.ConfVars.HIVE_IN_TEST_REPL.varname, "true"); + put(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, "true"); + + }}, "test_key"); + } + + @AfterClass + public static void classLevelTearDown() throws IOException { + primary.close(); + FileUtils.deleteQuietly(new File(jksFile)); + } + + @Before + public void setup() throws Throwable { + primaryDbName = testName.getMethodName() + "_" + +System.currentTimeMillis(); + primary.run("create database " + primaryDbName + " WITH DBPROPERTIES ( '" + + SOURCE_OF_REPLICATION + "' = '1,2,3')"); + } + + @Test + public void createTableEncryptionZoneRoot() throws Throwable { + boolean exceptionThrown = false; + try { + primary.run("use " + primaryDbName) + .run("create table encrypted_table (id int, value string) location '" + primary.getWarehouseRoot() + "'"); + } catch (HiveException e) { + exceptionThrown = true; + Assert.assertEquals("Table Location cannot be set to encryption zone root dir", e.getMessage()); + } + Assert.assertTrue(exceptionThrown); + + } + + @Test + public void createExternalTableEncryptionZoneRoot() throws Throwable { + boolean exceptionThrown = false; + try { + primary.run("use " + primaryDbName) + .run("create external table encrypted_external_table (id int, value string) location '" + + primary.getWarehouseRoot() + "'"); + } catch (HiveException e) { + exceptionThrown = true; + } finally { + primary.run("use " + primaryDbName) + .run("drop table encrypted_external_table"); + } + Assert.assertFalse(exceptionThrown); + } + + @Test + public void alterTableSetLocationEncryptionZoneRoot() throws Throwable { + boolean exceptionThrown = false; + try { + primary.run("use " + primaryDbName) + .run("create table encrypted_table (id int, value string) stored as orc TBLPROPERTIES('transactional'='true')"); + primary.run("use " + primaryDbName) + .run("alter table encrypted_table set location '" + primary.getWarehouseRoot() + "'"); + } catch (HiveException e) { + exceptionThrown = true; + Assert.assertEquals("Table Location cannot be set to encryption zone root dir", e.getMessage()); + } finally { + primary.run("use " + primaryDbName) + .run("drop table encrypted_table"); + } + Assert.assertTrue(exceptionThrown); + } + + @Test + public void alterExternalTableEncryptionZoneRoot() throws Throwable { + boolean exceptionThrown = false; + try { + primary.run("use " + primaryDbName) + .run("create external table encrypted_external_table (id int, value string)"); + primary.run("use " + primaryDbName) + .run("alter table encrypted_external_table set location '" + + primary.getWarehouseRoot() + "'"); + } catch (HiveException e) { + exceptionThrown = true; + } finally { + primary.run("use " + primaryDbName) + .run("drop table encrypted_external_table"); + } + Assert.assertFalse(exceptionThrown); + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java index 43effeb64e..a2ae3fcc8f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java @@ -91,8 +91,8 @@ private final static String LISTENER_CLASS = DbNotificationListener.class.getCanonicalName(); - WarehouseInstance(Logger logger, MiniDFSCluster cluster, Map overridesForHiveConf, - String keyNameForEncryptedZone) throws Exception { + public WarehouseInstance(Logger logger, MiniDFSCluster cluster, Map overridesForHiveConf, + String keyNameForEncryptedZone) throws Exception { this.logger = logger; this.miniDFSCluster = cluster; assert miniDFSCluster.isClusterUp(); @@ -640,6 +640,10 @@ CurrentNotificationEventId getCurrentNotificationEventId() throws Exception { return fileStatuses.stream().map(FileStatus::getPath).collect(Collectors.toList()); } + public Path getWarehouseRoot() { + return warehouseRoot; + } + static class Tuple { final String dumpLocation; final String lastReplicationId; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java index eb8b858a24..cc47ce8946 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java @@ -29,6 +29,7 @@ import java.util.TreeMap; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -45,6 +46,8 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.shims.HadoopShims; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.common.util.ReflectionUtil; import org.slf4j.Logger; @@ -219,4 +222,19 @@ private static String getHS2Host(HiveConf conf) throws SemanticException { throw new SemanticException("Kill query is only supported in HiveServer2 (not hive cli)"); } + + public static boolean isEncryptionZoneRoot(Path path, Configuration conf) throws IOException { + HadoopShims hadoopShims = ShimLoader.getHadoopShims(); + HadoopShims.HdfsEncryptionShim pathEncryptionShim + = hadoopShims.createHdfsEncryptionShim( + path.getFileSystem(conf), conf); + if (pathEncryptionShim.isPathEncrypted(path)) { + Path encryptionZoneRoot = new Path(path.getFileSystem(conf).getUri() + + pathEncryptionShim.getEncryptionZoneForPath(path).getPath()); + if (encryptionZoneRoot.equals(path)) { + return true; + } + } + return false; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java index 93c0209619..a0d8193367 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java @@ -59,6 +59,14 @@ public int execute() throws HiveException { LOG.debug("creating table {} on {}", tbl.getFullyQualifiedName(), tbl.getDataLocation()); boolean replDataLocationChanged = false; + try { + if (tbl.getSd().getLocation() != null && tbl.getTableType().equals(TableType.MANAGED_TABLE) + && DDLUtils.isEncryptionZoneRoot(new Path(tbl.getSd().getLocation()), context.getConf())) { + throw new HiveException("Table Location cannot be set to encryption zone root dir"); + } + } catch (IOException e) { + throw new HiveException(e); + } if (desc.getReplicationSpec().isInReplicationScope()) { // If in replication scope, we should check if the object we're looking at exists, and if so, // trigger replace-mode semantics. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/add/AlterTableAddPartitionOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/add/AlterTableAddPartitionOperation.java index 6910e100b0..7141568e50 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/add/AlterTableAddPartitionOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/add/AlterTableAddPartitionOperation.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.ddl.table.partition.add; +import java.io.IOException; import java.util.ArrayList; import java.util.BitSet; import java.util.List; @@ -26,6 +27,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; @@ -83,6 +85,15 @@ private long getWriteId(Table table) throws LockException { if (partition != null && writeId > 0) { partition.setWriteId(writeId); } + try { + if (partition != null && partition.getSd().getLocation() != null + && table.getTableType().equals(TableType.MANAGED_TABLE) + && DDLUtils.isEncryptionZoneRoot(new Path(partition.getSd().getLocation()), context.getConf())) { + throw new HiveException("Partition Location cannot be set to encryption zone root dir"); + } + } catch (IOException e) { + throw new HiveException(e); + } partitions.add(partition); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionsOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionsOperation.java index 52890952cb..2cb8cfa6d2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionsOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionsOperation.java @@ -18,9 +18,12 @@ package org.apache.hadoop.hive.ql.ddl.table.partition.exchange; +import java.io.IOException; import java.util.List; import java.util.Map; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.ddl.DDLOperation; import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; import org.apache.hadoop.hive.ql.ddl.DDLUtils; @@ -47,6 +50,14 @@ public int execute() throws HiveException { List partitions = context.getDb().exchangeTablePartitions(partitionSpecs, sourceTable.getDbName(), sourceTable.getTableName(), destTable.getDbName(), destTable.getTableName()); for (Partition partition : partitions) { + try { + if (partition.getLocation() != null && destTable.getTableType().equals(TableType.MANAGED_TABLE) + && DDLUtils.isEncryptionZoneRoot(new Path(partition.getLocation()), context.getConf())) { + throw new HiveException("Partition Location cannot be set to encryption zone root dir"); + } + } catch (IOException e) { + throw new HiveException(e); + } // Reuse the partition specs from dest partition since they should be the same context.getWork().getInputs().add(new ReadEntity(new Partition(sourceTable, partition.getSpec(), null))); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationOperation.java index 509d5770d3..abbacdec4b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationOperation.java @@ -18,14 +18,17 @@ package org.apache.hadoop.hive.ql.ddl.table.storage; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -49,8 +52,12 @@ protected void doAlteration(Table table, Partition partition) throws HiveExcepti if (!new Path(locUri).isAbsolute()) { throw new HiveException(ErrorMsg.BAD_LOCATION_VALUE, newLocation); } + if (table.getTableType().equals(TableType.MANAGED_TABLE) + && DDLUtils.isEncryptionZoneRoot(new Path(newLocation), context.getConf())) { + throw new HiveException("Table Location cannot be set to encryption zone root dir"); + } sd.setLocation(newLocation); - } catch (URISyntaxException e) { + } catch (URISyntaxException | IOException e) { throw new HiveException(e); } environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);