diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 957ebb1272..e3ac69c12f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -577,7 +577,7 @@ private void createTempTable(org.apache.hadoop.hive.metastore.api.Table tbl, ss.getTempTables().put(dbName, tables); } tables.put(tblName, tTable); - createTempTable(tbl); + createPartitionedTempTable(tbl); } private org.apache.hadoop.hive.metastore.api.Table getTempTable(String dbName, String tableName) @@ -774,7 +774,7 @@ private void dropTempTable(org.apache.hadoop.hive.metastore.api.Table table, boo throw new MetaException( "Could not find temp table entry for " + StatsUtils.getFullyQualifiedTableName(dbName, tableName)); } - removeTempTable(table); + removePartitionedTempTable(table); // Delete table data if (deleteData && !MetaStoreUtils.isExternalTable(table)) { @@ -927,8 +927,11 @@ private boolean deleteTempTableColumnStats(String dbName, String tableName, Stri this.tTable = t; pTree = t.getPartitionKeysSize() > 0 ? new PartitionTree(tTable) : null; } - private void addPartition(Partition p) throws AlreadyExistsException, MetaException { - pTree.addPartition(p); + + private Partition addPartition(Partition p) throws AlreadyExistsException, MetaException { + String partName = makePartName(tTable.getPartitionKeys(), p.getValues()); + Partition partition = pTree.addPartition(p, partName, false); + return partition == null ? pTree.getPartition(partName) : partition; } private Partition getPartition(String partName) throws MetaException { @@ -945,9 +948,9 @@ private Partition getPartition(List partVals) throws MetaException { return pTree.getPartition(partVals); } - private int addPartitions(List partitions) throws AlreadyExistsException, - MetaException { - return pTree.addPartitions(partitions); + private List addPartitions(List partitions, boolean ifNotExists) + throws MetaException, AlreadyExistsException { + return pTree.addPartitions(partitions, ifNotExists); } private List getPartitionsByNames(List partNames) throws MetaException { @@ -1041,11 +1044,14 @@ private boolean checkPrivilegesForPartition(Partition partition, String userName private PartitionTree(org.apache.hadoop.hive.metastore.api.Table t) { this.tTable = t; } - private void addPartition(Partition p) throws AlreadyExistsException, MetaException { - String partName = Warehouse.makePartName(tTable.getPartitionKeys(), p.getValues()); - if(parts.putIfAbsent(partName, p) != null) { + private Partition addPartition(Partition partition, String partName, boolean ifNotExists) + throws AlreadyExistsException { + partition.setDbName(partition.getDbName().toLowerCase()); + partition.setTableName(partition.getTableName().toLowerCase()); + if(!ifNotExists && parts.containsKey(partName)) { throw new AlreadyExistsException("Partition " + partName + " already exists"); } + return parts.putIfAbsent(partName, partition); } /** * @param partName - "p=1/q=2" full partition name {@link Warehouse#makePartName(List, List)} @@ -1068,16 +1074,35 @@ private Partition getPartition(List partVals) throws MetaException { return getPartition(partName); } - private int addPartitions(List partitions) - throws AlreadyExistsException, MetaException { - int partitionsAdded = 0; + /** + * Add partitions to the partition tree. + * + * @param partitions The partitions to add + * @param ifNotExists only add partitions if they don't exist + * @return the partitions that were added + * @throws MetaException + */ + private List addPartitions(List partitions, boolean ifNotExists) + throws MetaException, AlreadyExistsException { + List partitionsAdded = new ArrayList<>(); + Map partNameToPartition = new HashMap<>(); + // validate that the new partition values is not already added to the table for (Partition partition : partitions) { - addPartition(partition); - partitionsAdded++; + String partName = makePartName(tTable.getPartitionKeys(), partition.getValues()); + if (!ifNotExists && parts.containsKey(partName)) { + throw new AlreadyExistsException("Partition " + partName + " already exists"); + } + partNameToPartition.put(partName, partition); } + + for (Entry entry : partNameToPartition.entrySet()) { + if (addPartition(entry.getValue(), entry.getKey(), ifNotExists) == null) { + partitionsAdded.add(parts.get(entry.getKey())); + } + } + return partitionsAdded; } - /** * Provided values for the 1st N partition columns, will return all matching PartitionS * The list is a partial list of partition values in the same order as partition columns. @@ -1109,6 +1134,7 @@ private int addPartitions(List partitions) } } } + /** * Hive.loadPartition() calls this. * @param partition @@ -1119,6 +1145,9 @@ private int addPartitions(List partitions) public org.apache.hadoop.hive.metastore.api.Partition add_partition( org.apache.hadoop.hive.metastore.api.Partition partition) throws TException { // First try temp table + if (partition == null) { + throw new MetaException("Partition cannot be null"); + } org.apache.hadoop.hive.metastore.api.Table table = getTempTable(partition.getDbName(), partition.getTableName()); if (table == null) { @@ -1126,8 +1155,11 @@ private int addPartitions(List partitions) return super.add_partition(partition); } TempTable tt = getPartitionedTempTable(table); - tt.addPartition(deepCopy(partition)); - return partition; + checkPartitionProperties(partition); + Path partitionLocation = getPartitionLocation(table, partition); + Partition result = tt.addPartition(deepCopy(partition)); + createAndSetLocationForAddedPartition(result, partitionLocation); + return result; } /** @@ -1139,18 +1171,68 @@ private int addPartitions(List partitions) */ @Override public int add_partitions(List partitions) throws TException { + if (partitions == null || partitions.contains(null)) { + throw new MetaException("Partitions cannot be null"); + } if (partitions.isEmpty()) { return 0; } - Partition partition = partitions.get(0); + + List addedPartitions = add_partitions(partitions, false, true); + if (addedPartitions != null) { + return addedPartitions.size(); + } + + return super.add_partitions(partitions); + } + + @Override + public int add_partitions_pspec(PartitionSpecProxy partitionSpec) throws TException { + if (partitionSpec == null) { + throw new MetaException("PartitionSpec cannot be null."); + } + if (partitionSpec.size() == 0) { + return 0; + } + org.apache.hadoop.hive.metastore.api.Table table = - getTempTable(partition.getDbName(), partition.getTableName()); + getTempTable(partitionSpec.getDbName(), partitionSpec.getTableName()); if (table == null) { - // not a temp table - Try underlying client - return super.add_partitions(partitions); + return super.add_partitions_pspec(partitionSpec); } - TempTable tt = getPartitionedTempTable(table); - return tt.addPartitions(deepCopyPartitions(partitions)); + assertTempTablePartitioned(table); + PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpec.getPartitionIterator(); + List partitionsToAdd = new ArrayList<>(partitionSpec.size()); + while (partitionIterator.hasNext()) { + partitionsToAdd.add(partitionIterator.next()); + } + + List addedPartitions = addPartitionsToTempTable(partitionsToAdd, partitionSpec.getDbName(), + partitionSpec.getTableName(), false); + if (addedPartitions != null) { + return addedPartitions.size(); + } + + return super.add_partitions_pspec(partitionSpec); + } + + @Override + public List add_partitions(List partitions, boolean ifNotExists, boolean needResults) + throws TException { + if (partitions == null || partitions.contains(null)) { + throw new MetaException("Partitions cannot be null"); + } + + if (partitions.isEmpty()) { + return needResults ? new ArrayList<>() : null; + } + + List addedPartitions = addPartitionsToTempTable(partitions, null, null, ifNotExists); + if (addedPartitions != null) { + return needResults ? addedPartitions : null; + } + + return super.add_partitions(partitions, ifNotExists, needResults); } @@ -1375,7 +1457,7 @@ private TempTable getPartitionedTempTable(org.apache.hadoop.hive.metastore.api. LOG.warn("No current SessionState, skipping temp partitions for " + qualifiedTableName); return null; } - assertPartitioned(t); + assertTempTablePartitioned(t); TempTable tt = ss.getTempPartitions().get(qualifiedTableName); if (tt == null) { throw new IllegalStateException("TempTable not found for " + @@ -1383,7 +1465,7 @@ private TempTable getPartitionedTempTable(org.apache.hadoop.hive.metastore.api. } return tt; } - private void removeTempTable(org.apache.hadoop.hive.metastore.api.Table t) { + private void removePartitionedTempTable(org.apache.hadoop.hive.metastore.api.Table t) { String qualifiedTableName = Warehouse. getQualifiedName(t.getDbName().toLowerCase(), t.getTableName().toLowerCase()); SessionState ss = SessionState.get(); @@ -1393,7 +1475,8 @@ private void removeTempTable(org.apache.hadoop.hive.metastore.api.Table t) { } ss.getTempPartitions().remove(Warehouse.getQualifiedName(t)); } - private void createTempTable(org.apache.hadoop.hive.metastore.api.Table t) { + + private void createPartitionedTempTable(org.apache.hadoop.hive.metastore.api.Table t) { if(t.getPartitionKeysSize() <= 0) { //do nothing as it's not a partitioned table return; @@ -1411,7 +1494,195 @@ private void createTempTable(org.apache.hadoop.hive.metastore.api.Table t) { } } - private void assertPartitioned(org.apache.hadoop.hive.metastore.api.Table table) throws MetaException { + /** + * Create the directory for partition and set it to the partition. + * + * @param partition instance of the partition, must be not null + * @param partitionLocation the location of the partition + * @throws MetaException if the target directory already exists + */ + private void createAndSetLocationForAddedPartition(Partition partition, Path partitionLocation) throws MetaException { + + if (partitionLocation != null) { + partition.getSd().setLocation(partitionLocation.toString()); + + // Check to see if the directory already exists before calling + // mkdirs() because if the file system is read-only, mkdirs will + // throw an exception even if the directory already exists. + if (!wh.isDir(partitionLocation)) { + if (!wh.mkdirs(partitionLocation)) { + throw new MetaException(partitionLocation + + " is not a directory or unable to create one"); + } + } + } + } + + /** + * Checking the validity of some partition properties (values, storage descriptor, columns, serdeinfo). + * + * @param partition an instance of the partition, must be not null + * @throws MetaException if some check is failing + */ + private void checkPartitionProperties(Partition partition) throws MetaException { + if (partition.getDbName() == null) { + throw new MetaException("Database name cannot be null. " + partition); + } + if (partition.getTableName() == null) { + throw new MetaException("Table name cannot be null. " + partition); + } + if (partition.getValues() == null) { + throw new MetaException("Partition values cannot be null. " + partition); + } + if (partition.getSd() == null) { + throw new MetaException("Storage descriptor for partition cannot be null. " + partition); + } + if (partition.getSd().getCols() == null) { + // not sure this is correct, but it is possible to add a partition without column information. + return; + } + + for (FieldSchema schema : partition.getSd().getCols()) { + if (schema.getType() == null) { + throw new MetaException("Storage descriptor column type for partition cannot be null. " + partition); + } + if (schema.getName() == null) { + throw new MetaException("Storage descriptor column name for partition cannot be null. " + partition); + } + } + if (partition.getSd().getSerdeInfo() == null) { + throw new MetaException("Storage descriptor serde info for partition cannot be null. " + partition); + } + } + + /** + * Get the partition location. If the partition location is not set the location of the parent table will be used. + * + * @param table the parent table, must be not null + * @param partition instance of the partition, must be not null + * @return location of partition + * @throws MetaException if the partition location cannot be specified or the location is invalid. + */ + private Path getPartitionLocation(org.apache.hadoop.hive.metastore.api.Table table, Partition partition) + throws MetaException { + Path partLocation = null; + String partLocationStr = null; + if (partition.getSd() != null) { + partLocationStr = partition.getSd().getLocation(); + } + +// Warehouse wh = getWh(); + if (partLocationStr == null || partLocationStr.isEmpty()) { + // set default location if not specified and this is + // a physical table partition (not a view) + if (table.getSd().getLocation() != null) { + partLocation = new Path(table.getSd().getLocation(), + makePartName(table.getPartitionKeys(), partition.getValues())); + } + } else { + if (table.getSd().getLocation() == null) { + throw new MetaException("Cannot specify location for a view partition"); + } + try { + partLocation = wh.getDnsPath(new Path(partLocationStr)); + } catch (IllegalArgumentException e) { + throw new MetaException("Partition path is invalid. " + e.getLocalizedMessage()); + } + } + return partLocation; + } + + /** + * Try to add the partitions to an already existing temporary table. + * + * @param partitions The partitions to add. It must contain at least one item. + * @param dbName Name of the database, can be null + * @param tableName Name of the table, can be null + * @param ifNotExists only add partition if they don't exist + * @return the partitions that were added to the temp table. If the temp table is not found, null. + * @throws MetaException + */ + private List addPartitionsToTempTable(List partitions, String dbName, String tableName, + boolean ifNotExists) throws MetaException, AlreadyExistsException { + if (!validatePartitions(partitions, dbName, tableName)) { + return null; + } + if (dbName == null) { + dbName = partitions.get(0).getDbName(); + } + if (tableName == null) { + tableName = partitions.get(0).getTableName(); + } + org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName, tableName); + TempTable tt = getPartitionedTempTable(table); + List result = tt.addPartitions(deepCopyPartitions(partitions), ifNotExists); + for (Partition p : result) { + createAndSetLocationForAddedPartition(p, getPartitionLocation(table, p)); + } + return result; + } + + /** + * Validate various partition and table properties (dbName, tableName, partitionValues, table key size). + * @param partitions the partition, must be not null + * @param dbName name of the database, must be not null + * @param tableName name of the table, must be not null + * @return true, if all the validation passed. false, if the associated temporary table cannot be found + * @throws MetaException some validations can throw it. + */ + private boolean validatePartitions(List partitions, String dbName, String tableName) throws MetaException { + Set> partitionVals = new HashSet<>(); + // do some basic validation + for (Partition p : partitions) { + if (p.getDbName() == null) { + throw new MetaException("Database for partition cannot be null. " + p.toString()); + } + if (p.getTableName() == null) { + throw new MetaException("Table for partition cannot be null. " + p.toString()); + } + + // check that all new partitions are belonging to tables located in the same database + if (dbName != null && !dbName.equals(p.getDbName())) { + throw new MetaException("Partition tables doesn't belong to the same database " + + Arrays.toString(partitions.toArray())); + } else { + dbName = p.getDbName(); + } + // check if all new partitions are part of the same table + if (tableName != null && !tableName.equals(p.getTableName())) { + throw new MetaException("New partitions doesn't belong to the same table " + + Arrays.toString(partitions.toArray())); + } else { + tableName = p.getTableName(); + } + + // validate that new partitions belonging to the same table doesn't contain duplicate partition values + if (!partitionVals.contains(p.getValues())) { + partitionVals.add(p.getValues()); + } else { + throw new MetaException("Partition values contains duplicate entries. " + + Arrays.toString(partitions.toArray())); + } + + // check if all the tables are tmp tables + org.apache.hadoop.hive.metastore.api.Table table = getTempTable(p.getDbName(), p.getTableName()); + if (table == null) { + return false; + } + + checkPartitionProperties(p); + // validate partition location + getPartitionLocation(table, p); + } + return true; + } + + /** + * Check partition properties of temporary table. + * @param table table instance, must be not null. + * @throws MetaException if check fails. + */ + private void assertTempTablePartitioned(org.apache.hadoop.hive.metastore.api.Table table) throws MetaException { if(table.getPartitionKeysSize() <= 0) { throw new MetaException(getCatalogQualifiedTableName(table) + " is not partitioned"); } diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsFromSpecTempTable.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsFromSpecTempTable.java new file mode 100644 index 0000000000..59c665619e --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsFromSpecTempTable.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.CustomIgnoreRule; +import org.apache.hadoop.hive.metastore.client.TestAddPartitionsFromPartSpec; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Assert; +import org.junit.Before; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.List; + +/** + * Test class for adding partitions from partition spec related methods on temporary tables. + */ +@RunWith(Parameterized.class) +@Category(MetastoreCheckinTest.class) +public class TestSessionHiveMetastoreClientAddPartitionsFromSpecTempTable + extends TestAddPartitionsFromPartSpec { + + private HiveConf conf; + + public TestSessionHiveMetastoreClientAddPartitionsFromSpecTempTable(String name, AbstractMetaStoreService metaStore) { + super(name, metaStore); + ignoreRule = new CustomIgnoreRule(); + } + + @Before + public void setUp() throws Exception { + initHiveConf(); + SessionState.start(conf); + setClient(Hive.get(conf).getMSC()); + getClient().dropDatabase(DB_NAME, true, true, true); + getMetaStore().cleanWarehouseDirs(); + new DatabaseBuilder(). + setName(DB_NAME). + create(getClient(), conf); + } + + private void initHiveConf() throws HiveException { + conf = Hive.get().getConf(); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); + } + + @Override + protected Table createTable(String dbName, String tableName, List partCols, String location) + throws Exception { + new TableBuilder().setDbName(dbName).setTableName(tableName).addCol("test_id", "int", "test col id") + .addCol("test_value", "string", "test col value") + .addTableParam("partTestTableParamKey", "partTestTableParamValue").setPartCols(partCols) + .addStorageDescriptorParam("partTestSDParamKey", "partTestSDParamValue").setSerdeName(tableName) + .setStoredAsSubDirectories(false).addSerdeParam("partTestSerdeParamKey", "partTestSerdeParamValue") + .setLocation(location).setTemporary(true).create(getClient(), conf); + return getClient().getTable(dbName, tableName); + } + + @Override + protected void verifyPartition(Table table, String name, List values, int index) throws Exception { + + Partition part = getClient().getPartition(table.getDbName(), table.getTableName(), name); + Assert.assertNotNull("The partition should not be null.", part); + Assert.assertEquals("The table name in the partition is not correct.", table.getTableName(), part.getTableName()); + List partValues = part.getValues(); + Assert.assertEquals(values.size(), partValues.size()); + Assert.assertTrue("The partition has wrong values.", partValues.containsAll(values)); + Assert.assertEquals("The DB name in the partition is not correct.", table.getDbName(), part.getDbName()); + Assert.assertEquals("The last access time is not correct.", DEFAULT_CREATE_TIME, part.getLastAccessTime()); + Assert.assertEquals("The partition's parameter map should contain the partparamkey - partparamvalue pair.", + DEFAULT_PARAM_VALUE + index, part.getParameters().get(DEFAULT_PARAM_KEY + index)); + StorageDescriptor sd = part.getSd(); + Assert.assertNotNull("The partition's storage descriptor must not be null.", sd); + Assert.assertEquals("The input format is not correct.", "TestInputFormat" + index, sd.getInputFormat()); + Assert.assertEquals("The output format is not correct.", "TestOutputFormat" + index, sd.getOutputFormat()); + Assert.assertEquals("The serdeInfo name is not correct.", "partserde" + index, sd.getSerdeInfo().getName()); + Assert.assertEquals( + "The parameter map of the partition's storage descriptor should contain the partsdkey - partsdvalue pair.", + "partsdvalue" + index, sd.getParameters().get("partsdkey" + index)); + Assert.assertEquals("The parameter's location is not correct.", + getMetaStore().getWarehouseRoot() + "/" + TABLE_NAME + "/" + name, sd.getLocation()); + Assert.assertTrue("The parameter's location should exist on the file system.", + getMetaStore().isPathExists(new Path(sd.getLocation()))); + // If the 'metastore.partition.inherit.table.properties' property is set in the metastore + // config, the partition inherits the listed table parameters. + // This property is not set in this test, therefore the partition doesn't inherit the table + // parameters. + Assert.assertFalse("The partition should not inherit the table parameters.", + part.getParameters().keySet().contains(table.getParameters().keySet())); + } +} diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsTempTable.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsTempTable.java new file mode 100644 index 0000000000..f0b5193cab --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientAddPartitionsTempTable.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import com.google.common.collect.Lists; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.SkewedInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.client.CustomIgnoreRule; +import org.apache.hadoop.hive.metastore.client.TestAddPartitions; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Test class for adding partitions related methods on temporary tables. + */ +@RunWith(Parameterized.class) +@Category(MetastoreCheckinTest.class) +public class TestSessionHiveMetastoreClientAddPartitionsTempTable + extends TestAddPartitions { + + private HiveConf conf; + + public TestSessionHiveMetastoreClientAddPartitionsTempTable(String name, AbstractMetaStoreService metaStore) { + super(name, metaStore); + ignoreRule = new CustomIgnoreRule(); + } + + @Before + public void setUp() throws Exception { + initHiveConf(); + SessionState.start(conf); + setClient(Hive.get(conf).getMSC()); + getClient().dropDatabase(DB_NAME, true, true, true); + getMetaStore().cleanWarehouseDirs(); + new DatabaseBuilder(). + setName(DB_NAME). + create(getClient(), conf); + } + + private void initHiveConf() throws HiveException { + conf = Hive.get().getConf(); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); + } + + @Override + protected Table createTable(String dbName, String tableName, List partCols, String location) + throws Exception { + new TableBuilder().setDbName(dbName).setTableName(tableName).addCol("test_id", "int", "test col id") + .addCol("test_value", "string", "test col value") + .addTableParam("partTestTableParamKey", "partTestTableParamValue").setPartCols(partCols) + .addStorageDescriptorParam("partTestSDParamKey", "partTestSDParamValue").setSerdeName(tableName) + .setStoredAsSubDirectories(false).addSerdeParam("partTestSerdeParamKey", "partTestSerdeParamValue") + .setLocation(location).setTemporary(true).create(getClient(), conf); + return getClient().getTable(dbName, tableName); + } + + protected void createExternalTable(String tableName, String location) throws Exception { + new TableBuilder().setDbName(DB_NAME).setTableName(tableName).addCol("test_id", "int", "test col id") + .addCol("test_value", DEFAULT_COL_TYPE, "test col value").addPartCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) + .addTableParam("EXTERNAL", "TRUE").setLocation(location).setTemporary(true).create(getClient(), conf); + } + + @Override + protected void verifyPartition(Table table, String name, List values, int index) throws Exception { + + Partition part = getClient().getPartition(table.getDbName(), table.getTableName(), name); + Assert.assertNotNull("The partition should not be null.", part); + assertEquals("The table name in the partition is not correct.", table.getTableName(), part.getTableName()); + List partValues = part.getValues(); + assertEquals(values.size(), partValues.size()); + Assert.assertTrue("The partition has wrong values.", partValues.containsAll(values)); + assertEquals("The DB name in the partition is not correct.", table.getDbName(), part.getDbName()); + assertEquals("The last access time is not correct.", 123456, part.getLastAccessTime()); + assertEquals("The partition's parameter map should contain the partparamkey - partparamvalue pair.", + DEFAULT_PARAM_VALUE + index, part.getParameters().get(DEFAULT_PARAM_KEY + index)); + StorageDescriptor sd = part.getSd(); + Assert.assertNotNull("The partition's storage descriptor must not be null.", sd); + assertEquals("The input format is not correct.", "TestInputFormat" + index, sd.getInputFormat()); + assertEquals("The output format is not correct.", "TestOutputFormat" + index, sd.getOutputFormat()); + assertEquals("The serdeInfo name is not correct.", "partserde" + index, sd.getSerdeInfo().getName()); + assertEquals( + "The parameter map of the partition's storage descriptor should contain the partsdkey - partsdvalue pair.", + "partsdvalue" + index, sd.getParameters().get("partsdkey" + index)); + assertEquals("The parameter's location is not correct.", + getMetaStore().getWarehouseRoot() + "/" + TABLE_NAME + "/" + name, sd.getLocation()); + Assert.assertTrue("The parameter's location should exist on the file system.", + getMetaStore().isPathExists(new Path(sd.getLocation()))); + Assert.assertFalse("The partition should not inherit the table parameters.", + part.getParameters().keySet().contains(table.getParameters().keySet())); + } + + @Override + protected void verifyPartitionAttributesDefaultValues(Partition partition, String tableLocation) { + Assert.assertNotEquals("The partition's last access time should be set.", 0, partition.getLastAccessTime()); + Assert.assertNotEquals("The partition's create time should be set.", 0, partition.getCreateTime()); + StorageDescriptor sd = partition.getSd(); + Assert.assertNotNull("The storage descriptor of the partition must not be null.", sd); + assertEquals("The partition location is not correct.", tableLocation + "/year=2017", sd.getLocation()); + assertEquals("The input format doesn't have the default value.", "org.apache.hadoop.hive.ql.io.HiveInputFormat", + sd.getInputFormat()); + assertEquals("The output format doesn't have the default value.", "org.apache.hadoop.hive.ql.io.HiveOutputFormat", + sd.getOutputFormat()); + Assert.assertFalse("The compressed attribute doesn't have the default value.", sd.isCompressed()); + Assert.assertFalse("The storedAsSubDirectories attribute doesn't have the default value.", + sd.isStoredAsSubDirectories()); + assertEquals("The numBuckets attribute doesn't have the default value.", 0, sd.getNumBuckets()); + Assert.assertTrue("The default value of the attribute 'bucketCols' should be an empty list.", + sd.getBucketCols().isEmpty()); + Assert.assertTrue("The default value of the attribute 'sortCols' should be an empty list.", + sd.getSortCols().isEmpty()); + Assert.assertTrue("Per default the storage descriptor parameters should be empty.", sd.getParameters().isEmpty()); + SerDeInfo serdeInfo = sd.getSerdeInfo(); + Assert.assertNotNull("The serdeInfo attribute should not be null.", serdeInfo); + Assert.assertNull("The default value of the serde's name attribute should be null.", serdeInfo.getName()); + assertEquals("The serde's 'serializationLib' attribute doesn't have the default value.", + "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", serdeInfo.getSerializationLib()); + Assert.assertTrue("Per default the serde info parameters should be empty.", serdeInfo.getParameters().isEmpty()); + SkewedInfo skewedInfo = sd.getSkewedInfo(); + Assert.assertTrue("Per default the skewedInfo column names list should be empty.", + skewedInfo.getSkewedColNames().isEmpty()); + Assert.assertTrue("Per default the skewedInfo column value list should be empty.", + skewedInfo.getSkewedColValues().isEmpty()); + Assert.assertTrue("Per default the skewedInfo column value location map should be empty.", + skewedInfo.getSkewedColValueLocationMaps().isEmpty()); + } + + @Test + @Override + public void testAddPartitionNullLocationInTableToo() throws Exception { + createTable(DB_NAME, TABLE_NAME, null); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE, null); + getClient().add_partition(partition); + Partition part = getClient().getPartition(DB_NAME, TABLE_NAME, "year=2017"); + Assert.assertTrue(getMetaStore().isPathExists(new Path(part.getSd().getLocation()))); + assertEquals(SessionState.get().getTempTables().get(DB_NAME).get(TABLE_NAME).getSd().getLocation() + "/year=2017", + part.getSd().getLocation()); + } + + @Test + @Override + public void testAddPartitionForExternalTableNullLocation() throws Exception { + String tableName = "part_add_ext_table"; + createExternalTable(tableName, null); + Partition partition = buildPartition(DB_NAME, tableName, DEFAULT_YEAR_VALUE, null); + getClient().add_partition(partition); + Partition resultPart = getClient().getPartition(DB_NAME, tableName, Lists.newArrayList(DEFAULT_YEAR_VALUE)); + Assert.assertNotNull(resultPart); + Assert.assertNotNull(resultPart.getSd()); + assertEquals(SessionState.get().getTempTables().get(DB_NAME).get(tableName).getSd().getLocation() + "/year=2017", + resultPart.getSd().getLocation()); + } + + @Test + @Override + public void testAddPartitionsNullLocationInTableToo() throws Exception { + createTable(DB_NAME, TABLE_NAME, null); + List partitions = new ArrayList<>(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE, null); + partitions.add(partition); + getClient().add_partitions(partitions); + + Partition part = getClient().getPartition(DB_NAME, TABLE_NAME, "year=2017"); + assertEquals(SessionState.get().getTempTables().get(DB_NAME).get(TABLE_NAME).getSd().getLocation() + "/year=2017", + part.getSd().getLocation()); + Assert.assertTrue(getMetaStore().isPathExists(new Path(part.getSd().getLocation()))); + } + + @Test + @Override + public void testAddPartitionsForExternalTableNullLocation() throws Exception { + String tableName = "part_add_ext_table"; + createExternalTable(tableName, null); + Partition partition1 = buildPartition(DB_NAME, tableName, "2017", null); + Partition partition2 = buildPartition(DB_NAME, tableName, "2018", null); + List partitions = Lists.newArrayList(partition1, partition2); + getClient().add_partitions(partitions); + + List resultParts = + getClient().getPartitionsByNames(DB_NAME, tableName, Lists.newArrayList("year=2017", "year=2018")); + Assert.assertNotNull(resultParts); + Assert.assertEquals(2, resultParts.size()); + String defaultTableLocation = SessionState.get().getTempTables().get(DB_NAME).get(tableName).getSd().getLocation(); + String defaultPartLocation1 = defaultTableLocation + "/year=2017"; + String defaultPartLocation2 = defaultTableLocation + "/year=2018"; + if (resultParts.get(0).getValues().get(0).equals("2017")) { + Assert.assertEquals(defaultPartLocation1, resultParts.get(0).getSd().getLocation()); + Assert.assertEquals(defaultPartLocation2, resultParts.get(1).getSd().getLocation()); + } else { + Assert.assertEquals(defaultPartLocation2, resultParts.get(0).getSd().getLocation()); + Assert.assertEquals(defaultPartLocation1, resultParts.get(1).getSd().getLocation()); + } + } +} diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java index a15f5ea045..3d66c3f027 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java @@ -65,13 +65,13 @@ private AbstractMetaStoreService metaStore; private IMetaStoreClient client; - private static final String DB_NAME = "test_partition_db"; - private static final String TABLE_NAME = "test_partition_table"; - private static final String DEFAULT_PARAM_VALUE = "partparamvalue"; - private static final String DEFAULT_PARAM_KEY = "partparamkey"; - private static final String DEFAULT_YEAR_VALUE = "2017"; - private static final String DEFAULT_COL_TYPE = "string"; - private static final String YEAR_COL_NAME = "year"; + protected static final String DB_NAME = "test_partition_db"; + protected static final String TABLE_NAME = "test_partition_table"; + protected static final String DEFAULT_PARAM_VALUE = "partparamvalue"; + protected static final String DEFAULT_PARAM_KEY = "partparamkey"; + protected static final String DEFAULT_YEAR_VALUE = "2017"; + protected static final String DEFAULT_COL_TYPE = "string"; + protected static final String YEAR_COL_NAME = "year"; private static final String MONTH_COL_NAME = "month"; private static final short MAX = -1; @@ -105,6 +105,18 @@ public void tearDown() throws Exception { } } + protected AbstractMetaStoreService getMetaStore() { + return metaStore; + } + + protected IMetaStoreClient getClient() { + return client; + } + + protected void setClient(IMetaStoreClient client) { + this.client = client; + } + // Tests for the Partition add_partition(Partition partition) method @Test @@ -130,6 +142,7 @@ public void testAddPartitionTwoValues() throws Exception { } @Test + @ConditionalIgnoreOnSessionHiveMetastoreClient public void addPartitionOtherCatalog() throws TException { String catName = "add_partition_catalog"; Catalog cat = new CatalogBuilder() @@ -177,6 +190,7 @@ public void addPartitionOtherCatalog() throws TException { } @Test(expected = InvalidObjectException.class) + @ConditionalIgnoreOnSessionHiveMetastoreClient public void noSuchCatalog() throws TException { String tableName = "table_for_no_such_catalog"; Table table = new TableBuilder() @@ -462,6 +476,7 @@ public void testAddPartitionNullLocationInTableToo() throws Exception { } @Test(expected = MetaException.class) + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testAddPartitionForView() throws Exception { String tableName = "test_add_partition_view"; @@ -471,6 +486,7 @@ public void testAddPartitionForView() throws Exception { } @Test + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testAddPartitionsForViewNullPartLocation() throws Exception { String tableName = "test_add_partition_view"; @@ -484,6 +500,7 @@ public void testAddPartitionsForViewNullPartLocation() throws Exception { } @Test + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testAddPartitionsForViewNullPartSd() throws Exception { String tableName = "test_add_partition_view"; @@ -770,6 +787,7 @@ public void testAddPartitionsUpperCaseDBAndTableNameInOnePart() throws Exception Assert.fail("MetaException should have been thrown."); } catch (MetaException e) { // Expected exception + System.out.println(e); } List partitionNames = client.listPartitionNames(DB_NAME, tableName, MAX); @@ -810,7 +828,6 @@ public void testAddPartitionsDifferentTable() throws Exception { @Test public void testAddPartitionsDifferentDBs() throws Exception { - createDB("parttestdb2"); createTable(); createTable("parttestdb2", TABLE_NAME, null); @@ -1148,6 +1165,7 @@ public void testAddPartitionsNullLocationInTableToo() throws Exception { } @Test(expected=MetaException.class) + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testAddPartitionsForView() throws Exception { String tableName = "test_add_partition_view"; @@ -1513,11 +1531,11 @@ private Table createTable() throws Exception { return createTable(DB_NAME, TABLE_NAME, metaStore.getWarehouseRoot() + "/" + TABLE_NAME); } - private Table createTable(String dbName, String tableName, String location) throws Exception { + protected Table createTable(String dbName, String tableName, String location) throws Exception { return createTable(dbName, tableName, getYearPartCol(), location); } - private Table createTable(String dbName, String tableName, List partCols, + protected Table createTable(String dbName, String tableName, List partCols, String location) throws Exception { new TableBuilder() .setDbName(dbName) @@ -1535,7 +1553,7 @@ private Table createTable(String dbName, String tableName, List par return client.getTable(dbName, tableName); } - private void createExternalTable(String tableName, String location) throws Exception { + protected void createExternalTable(String tableName, String location) throws Exception { new TableBuilder() .setDbName(DB_NAME) .setTableName(tableName) @@ -1547,13 +1565,13 @@ private void createExternalTable(String tableName, String location) throws Excep .create(client, metaStore.getConf()); } - private Partition buildPartition(String dbName, String tableName, String value) + protected Partition buildPartition(String dbName, String tableName, String value) throws MetaException { return buildPartition(dbName, tableName, value, metaStore.getWarehouseRoot() + "/" + tableName + "/addparttest"); } - private Partition buildPartition(String dbName, String tableName, String value, + protected Partition buildPartition(String dbName, String tableName, String value, String location) throws MetaException { Partition partition = new PartitionBuilder() .setDbName(dbName) @@ -1607,7 +1625,7 @@ private Partition buildPartition(List values, List partCols return cols; } - private void verifyPartition(Table table, String name, List values, int index) + protected void verifyPartition(Table table, String name, List values, int index) throws Exception { Partition part = client.getPartition(table.getDbName(), table.getTableName(), name); @@ -1647,7 +1665,7 @@ private void verifyPartition(Table table, String name, List values, int part.getParameters().keySet().contains(table.getParameters().keySet())); } - private void verifyPartitionAttributesDefaultValues(Partition partition, String tableLocation) { + protected void verifyPartitionAttributesDefaultValues(Partition partition, String tableLocation) { Assert.assertNotEquals("The partition's last access time should be set.", 0, partition.getLastAccessTime()); Assert.assertNotEquals("The partition's create time should be set.", 0, diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java index 25643495b5..e0662bf6a3 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java @@ -63,15 +63,15 @@ private AbstractMetaStoreService metaStore; private IMetaStoreClient client; - private static final String DB_NAME = "test_partition_db"; - private static final String TABLE_NAME = "test_partition_table"; - private static final String DEFAULT_PARAM_VALUE = "partparamvalue"; - private static final String DEFAULT_PARAM_KEY = "partparamkey"; + protected static final String DB_NAME = "test_partition_db"; + protected static final String TABLE_NAME = "test_partition_table"; + protected static final String DEFAULT_PARAM_VALUE = "partparamvalue"; + protected static final String DEFAULT_PARAM_KEY = "partparamkey"; private static final String DEFAULT_YEAR_VALUE = "2017"; private static final String DEFAULT_COL_TYPE = "string"; private static final String YEAR_COL_NAME = "year"; private static final String MONTH_COL_NAME = "month"; - private static final int DEFAULT_CREATE_TIME = 123456; + protected static final int DEFAULT_CREATE_TIME = 123456; private static final short MAX = -1; public TestAddPartitionsFromPartSpec(String name, AbstractMetaStoreService metaStore) { @@ -106,6 +106,18 @@ public void tearDown() throws Exception { } } + protected AbstractMetaStoreService getMetaStore() { + return metaStore; + } + + protected IMetaStoreClient getClient() { + return client; + } + + protected void setClient(IMetaStoreClient client) { + this.client = client; + } + // Tests for int add_partitions_pspec(PartitionSpecProxy partitionSpec) method @Test @@ -238,6 +250,7 @@ public void testAddPartitionSpecUpperCaseDBAndTableName() throws Exception { } @Test + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testAddPartitionSpecUpperCaseDBAndTableNameInOnePart() throws Exception { String tableName = "test_add_part_table"; @@ -702,6 +715,7 @@ public void testAddPartitionSpecWithSharedSDEmptyLocation() throws Exception { } @Test(expected = MetaException.class) + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testAddPartitionSpecWithSharedSDInvalidSD() throws Exception { Table table = createTable(); @@ -763,6 +777,7 @@ public void testAddPartitionSpecEmptyLocationInTableToo() throws Exception { } @Test(expected=MetaException.class) + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testAddPartitionSpecForView() throws Exception { String tableName = "test_add_partition_view"; @@ -774,6 +789,7 @@ public void testAddPartitionSpecForView() throws Exception { } @Test + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testAddPartitionSpecForViewNullPartLocation() throws Exception { String tableName = "test_add_partition_view"; @@ -788,6 +804,7 @@ public void testAddPartitionSpecForViewNullPartLocation() throws Exception { } @Test + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testAddPartitionsForViewNullPartSd() throws Exception { String tableName = "test_add_partition_view"; @@ -1005,7 +1022,7 @@ private Table createTable() throws Exception { metaStore.getWarehouseRoot() + "/" + TABLE_NAME); } - private Table createTable(String dbName, String tableName, List partCols, + protected Table createTable(String dbName, String tableName, List partCols, String location) throws Exception { new TableBuilder() .setDbName(dbName) @@ -1077,7 +1094,7 @@ private Partition buildPartition(List values, List partCols return cols; } - private void verifyPartition(Table table, String name, List values, int index) + protected void verifyPartition(Table table, String name, List values, int index) throws Exception { Partition part = client.getPartition(table.getDbName(), table.getTableName(), name);