diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java index 09be321..106d0bd 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java @@ -19,7 +19,9 @@ package org.apache.hadoop.hive.metastore.client; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -31,6 +33,10 @@ import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionListComposingSpec; +import org.apache.hadoop.hive.metastore.api.PartitionSpec; +import org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD; +import org.apache.hadoop.hive.metastore.api.PartitionWithoutSD; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -39,6 +45,7 @@ import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.thrift.TException; import org.apache.thrift.transport.TTransportException; import org.junit.After; @@ -1299,6 +1306,720 @@ public void testAddPartsNullPartition() throws Exception { client.add_partitions(partitions, false, false); } + // Tests for int add_partitions_pspec(PartitionSpecProxy partitionSpec) method + + @Test + public void testAddPartitionSpec() throws Exception { + + Table table = createTable(); + Partition partition1 = buildPartition(Lists.newArrayList("2013"), getYearPartCol(), 1); + Partition partition2 = buildPartition(Lists.newArrayList("2014"), getYearPartCol(), 2); + Partition partition3 = buildPartition(Lists.newArrayList("2012"), getYearPartCol(), 3); + List partitions = Lists.newArrayList(partition1, partition2, partition3); + + String rootPath = table.getSd().getLocation() + "/addpartspectest/"; + PartitionSpecProxy partitionSpec = + buildPartitionSpec(DB_NAME, TABLE_NAME, rootPath, partitions); + client.add_partitions_pspec(partitionSpec); + + verifyPartition(table, "year=2013", Lists.newArrayList("2013"), 1); + verifyPartition(table, "year=2014", Lists.newArrayList("2014"), 2); + verifyPartition(table, "year=2012", Lists.newArrayList("2012"), 3); + } + + @Test + public void testAddPartitionSpecWithSharedSD() throws Exception { + + Table table = createTable(); + PartitionWithoutSD partition1 = buildPartitionWithoutSD(Lists.newArrayList("2013"), 1); + PartitionWithoutSD partition2 = buildPartitionWithoutSD(Lists.newArrayList("2014"), 2); + PartitionWithoutSD partition3 = buildPartitionWithoutSD(Lists.newArrayList("2012"), 3); + List partitions = Lists.newArrayList(partition1, partition2, partition3); + + String location = table.getSd().getLocation() + "/sharedSDTest/"; + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpecWithSharedSD(partitions, buildSD(location)); + client.add_partitions_pspec(partitionSpecProxy); + + verifyPartitionSharedSD(table, "year=2013", Lists.newArrayList("2013"), 1); + verifyPartitionSharedSD(table, "year=2014", Lists.newArrayList("2014"), 2); + verifyPartitionSharedSD(table, "year=2012", Lists.newArrayList("2012"), 3); + } + + @Test + public void testAddPartitionSpecsMultipleValues() throws Exception { + + Table table = createTable(DB_NAME, TABLE_NAME, getYearAndMonthPartCols(), + metaStore.getWarehouseRoot() + "/" + TABLE_NAME); + + Partition partition1 = + buildPartition(Lists.newArrayList("2002", "march"), getYearAndMonthPartCols(), 1); + Partition partition2 = + buildPartition(Lists.newArrayList("2003", "april"), getYearAndMonthPartCols(), 2); + PartitionWithoutSD partition3 = buildPartitionWithoutSD(Lists.newArrayList("2004", "june"), 3); + PartitionWithoutSD partition4 = buildPartitionWithoutSD(Lists.newArrayList("2005", "may"), 4); + List partitions = Lists.newArrayList(partition1, partition2); + List partitionsWithoutSD = Lists.newArrayList(partition3, partition4); + + PartitionSpecProxy partitionSpec = buildPartitionSpec(partitions, partitionsWithoutSD); + client.add_partitions_pspec(partitionSpec); + + verifyPartition(table, "year=2002/month=march", Lists.newArrayList("2002", "march"), 1); + verifyPartition(table, "year=2003/month=april", Lists.newArrayList("2003", "april"), 2); + verifyPartitionSharedSD(table, "year=2004/month=june", Lists.newArrayList("2004", "june"), 3); + verifyPartitionSharedSD(table, "year=2005/month=may", Lists.newArrayList("2005", "may"), 4); + } + + @Test(expected = NullPointerException.class) + public void testAddPartitionSpecNullSpec() throws Exception { + + // TODO: NPE should not be thrown. + client.add_partitions_pspec(null); + } + + @Test + public void testAddPartitionSpecEmptyPartList() throws Exception { + + createTable(); + List partitions = new ArrayList<>(); + PartitionSpecProxy partitionSpec = buildPartitionSpec(DB_NAME, TABLE_NAME, null, partitions); + client.add_partitions_pspec(partitionSpec); + } + + @Test + public void testAddPartitionSpecNullPartList() throws Exception { + + createTable(); + List partitions = null; + PartitionSpecProxy partitionSpec = buildPartitionSpec(DB_NAME, TABLE_NAME, null, partitions); + try { + client.add_partitions_pspec(partitionSpec); + Assert.fail("Exception should have been thrown."); + } catch (NullPointerException | TTransportException e) { + // TODO: NPE should not be thrown. + } + } + + @Test + public void testAddPartitionSpecNoDB() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(null, TABLE_NAME, null, Lists.newArrayList(partition)); + try { + client.add_partitions_pspec(partitionSpecProxy); + Assert.fail("Exception should have been thrown."); + } catch (NullPointerException | TTransportException e) { + // TODO: NPE should not be thrown. + } + } + + @Test + public void testAddPartitionSpecNoTable() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, null, null, Lists.newArrayList(partition)); + try { + client.add_partitions_pspec(partitionSpecProxy); + Assert.fail("Exception should have been thrown."); + } catch (NullPointerException | TTransportException e) { + // TODO: NPE should not be thrown. + } + } + + @Test + public void testAddPartitionSpecNoDBAndTableInPartition() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + partition.setDbName(null); + partition.setTableName(null); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + try { + client.add_partitions_pspec(partitionSpecProxy); + Assert.fail("Exception should have been thrown."); + } catch (NullPointerException | TTransportException e) { + // TODO: NPE should not be thrown. + } + } + + @Test + public void testAddPartitionSpecDBAndTableSetFromSpecProxy() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + partition.setDbName(null); + partition.setTableName(null); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(null, null, null, Lists.newArrayList(partition)); + partitionSpecProxy.setDbName(DB_NAME); + partitionSpecProxy.setTableName(TABLE_NAME); + client.add_partitions_pspec(partitionSpecProxy); + + Partition resultPart = + client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList(DEFAULT_YEAR_VALUE)); + Assert.assertNotNull(resultPart); + } + + @Test + public void testAddPartitionSpecWithSharedSDDBAndTableSetFromSpecProxy() throws Exception { + + createTable(); + PartitionWithoutSD partition = + buildPartitionWithoutSD(Lists.newArrayList(DEFAULT_YEAR_VALUE), 1); + String location = metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/sharedSDTest/"; + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), buildSD(location)); + partitionSpecProxy.setDbName(DB_NAME); + partitionSpecProxy.setTableName(TABLE_NAME); + client.add_partitions_pspec(partitionSpecProxy); + + Partition resultPart = + client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList(DEFAULT_YEAR_VALUE)); + Assert.assertNotNull(resultPart); + } + + @Test(expected = InvalidObjectException.class) + public void testAddPartitionSpecEmptyDB() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec("", TABLE_NAME, null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test(expected = InvalidObjectException.class) + public void testAddPartitionSpecEmptyTable() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, "", null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test(expected = InvalidObjectException.class) + public void testAddPartitionSpecNonExistingDB() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec("nonexistingdb", TABLE_NAME, null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test(expected = InvalidObjectException.class) + public void testAddPartitionSpecNonExistingTable() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, "nonexistingtable", null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test + public void testAddPartitionSpecDiffDBName() throws Exception { + + createDB("NewPartDB"); + createTable(); + createTable("NewPartDB", "NewPartTable", null); + List partitions = new ArrayList<>(); + Partition partition1 = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + Partition partition2 = buildPartition("NewPartDB", "NewPartTable", DEFAULT_YEAR_VALUE); + partitions.add(partition1); + partitions.add(partition2); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, partitions); + try { + client.add_partitions_pspec(partitionSpecProxy); + Assert.fail("MetaException should have been thrown."); + } catch (MetaException e) { + // Expected exception + } finally { + client.dropDatabase("NewPartDB", true, true, true); + } + } + + @Test + public void testAddPartitionSpecNullPart() throws Exception { + + createTable(); + List partitions = new ArrayList<>(); + Partition partition1 = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + Partition partition2 = null; + partitions.add(partition1); + partitions.add(partition2); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, partitions); + try { + client.add_partitions_pspec(partitionSpecProxy); + } catch (NullPointerException e) { + // TODO: NPE should not be thrown. + } + } + + @Test + public void testAddPartitionSpecUnsupportedPartSpecType() throws Exception { + + createTable(); + PartitionSpec partitionSpec = new PartitionSpec(); + partitionSpec.setDbName(DB_NAME); + partitionSpec.setTableName(TABLE_NAME); + partitionSpec.setPartitionList(null); + partitionSpec.setSharedSDPartitionSpec(null); + try { + PartitionSpecProxy bubu = PartitionSpecProxy.Factory.get(partitionSpec); + client.add_partitions_pspec(bubu); + Assert.fail("AssertionError should have been thrown."); + } catch (AssertionError e) { + // Expected error + } + } + + @Test + public void testAddPartitionSpecBothTypeSet() throws Exception { + + Table table = createTable(); + Partition partition = buildPartition(Lists.newArrayList("2013"), getYearPartCol(), 1); + PartitionWithoutSD partitionWithoutSD = buildPartitionWithoutSD(Lists.newArrayList("2014"), 0); + + PartitionSpec partitionSpec = new PartitionSpec(); + partitionSpec.setDbName(DB_NAME); + partitionSpec.setTableName(TABLE_NAME); + PartitionListComposingSpec partitionListComposingSpec = new PartitionListComposingSpec(); + partitionListComposingSpec.setPartitions(Lists.newArrayList(partition)); + partitionSpec.setPartitionList(partitionListComposingSpec); + + PartitionSpecWithSharedSD partitionSpecWithSharedSD = new PartitionSpecWithSharedSD(); + partitionSpecWithSharedSD.setPartitions(Lists.newArrayList(partitionWithoutSD)); + partitionSpecWithSharedSD.setSd(buildSD(table.getSd().getLocation() + "/sharedSDTest/")); + partitionSpec.setSharedSDPartitionSpec(partitionSpecWithSharedSD); + + PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(partitionSpec); + client.add_partitions_pspec(partitionSpecProxy); + + List partitionNames = client.listPartitionNames(DB_NAME, TABLE_NAME, MAX); + Assert.assertNotNull(partitionNames); + Assert.assertTrue(partitionNames.size() == 1); + Assert.assertEquals("year=2013", partitionNames.get(0)); + } + + @Test + public void testAddPartitionSpecSetRootPath() throws Exception { + + Table table = createTable(); + String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; + String rootPath1 = table.getSd().getLocation() + "/someotherpath/"; + + Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath + "part2007/"); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, rootPath1, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + + Partition resultPart = client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList("2007")); + Assert.assertEquals(rootPath + "part2007", resultPart.getSd().getLocation()); + } + + @Test + public void testAddPartitionSpecChangeRootPath() throws Exception { + + Table table = createTable(); + String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; + String rootPath1 = table.getSd().getLocation() + "/someotherpath/"; + + Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath + "part2007/"); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, rootPath, Lists.newArrayList(partition)); + partitionSpecProxy.setRootLocation(rootPath1); + client.add_partitions_pspec(partitionSpecProxy); + + Partition resultPart = client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList("2007")); + Assert.assertEquals(rootPath1 + "part2007", resultPart.getSd().getLocation()); + } + + @Test(expected = MetaException.class) + public void testAddPartitionSpecChangeRootPathFromNull() throws Exception { + + Table table = createTable(); + String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; + String rootPath1 = table.getSd().getLocation() + "/someotherpath/"; + + Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath + "part2007/"); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + partitionSpecProxy.setRootLocation(rootPath1); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test(expected = NullPointerException.class) + public void testAddPartitionSpecChangeRootPathToNull() throws Exception { + + Table table = createTable(); + String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; + Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath + "part2007/"); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, rootPath, Lists.newArrayList(partition)); + partitionSpecProxy.setRootLocation(null); + client.add_partitions_pspec(partitionSpecProxy); + // TODO: NPE should not be thrown. + } + + @Test(expected = MetaException.class) + public void testAddPartitionSpecChangeRootPathDiffInSd() throws Exception { + + Table table = createTable(); + String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; + String rootPath1 = table.getSd().getLocation() + "/addPartSdPath/"; + String rootPath2 = table.getSd().getLocation() + "/someotherpath/"; + Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath1 + "part2007/"); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, rootPath, Lists.newArrayList(partition)); + partitionSpecProxy.setRootLocation(rootPath2); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test + public void testAddPartitionSpecWithSharedSDChangeRootPath() throws Exception { + + Table table = createTable(); + String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; + String rootPath1 = table.getSd().getLocation() + "/someotherpath/"; + + PartitionWithoutSD partition = buildPartitionWithoutSD(Lists.newArrayList("2014"), 0); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), buildSD(rootPath)); + partitionSpecProxy.setRootLocation(rootPath1); + client.add_partitions_pspec(partitionSpecProxy); + + Partition resultPart = client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList("2014")); + Assert.assertEquals(rootPath1 + "partwithoutsd0", resultPart.getSd().getLocation()); + } + + @Test + public void testAddPartitionSpecWithSharedSDWithoutRelativePath() throws Exception { + + Table table = createTable(); + PartitionWithoutSD partition = buildPartitionWithoutSD(Lists.newArrayList("2014"), 0); + partition.setRelativePath(null); + String location = table.getSd().getLocation() + "/sharedSDTest/"; + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), buildSD(location)); + client.add_partitions_pspec(partitionSpecProxy); + + Partition part = client.getPartition(DB_NAME, TABLE_NAME, "year=2014"); + Assert.assertNotNull(part); + Assert.assertEquals(table.getSd().getLocation() + "/sharedSDTest/null", + part.getSd().getLocation()); + Assert.assertTrue(metaStore.isPathExists(new Path(part.getSd().getLocation()))); + } + + @Test(expected = AlreadyExistsException.class) + public void testAddPartitionSpecPartAlreadyExists() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + client.add_partition(partition); + + Partition newPartition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(newPartition)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test(expected = MetaException.class) + public void testAddPartitionSpecPartDuplicateInSpec() throws Exception { + + createTable(); + Partition partition1 = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + Partition partition2 = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + List partitions = new ArrayList<>(); + partitions.add(partition1); + partitions.add(partition2); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, partitions); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test(expected = MetaException.class) + public void testAddPartitionSpecPartDuplicateInSpecs() throws Exception { + + createTable(DB_NAME, TABLE_NAME, getYearPartCol(), + metaStore.getWarehouseRoot() + "/" + TABLE_NAME); + + Partition partition = buildPartition(Lists.newArrayList("2002"), getYearPartCol(), 1); + PartitionWithoutSD partitionWithoutSD = buildPartitionWithoutSD(Lists.newArrayList("2002"), 0); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(Lists.newArrayList(partition), Lists.newArrayList(partitionWithoutSD)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test(expected = MetaException.class) + public void testAddPartitionSpecNullSd() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + partition.setSd(null); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test + public void testAddPartitionSpecWithSharedSDNullSd() throws Exception { + + createTable(); + PartitionWithoutSD partition = buildPartitionWithoutSD(Lists.newArrayList("2002"), 0); + StorageDescriptor sd = null; + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), sd); + try { + client.add_partitions_pspec(partitionSpecProxy); + Assert.fail("Exception should have been thrown."); + } catch (NullPointerException | TTransportException e) { + // TODO: NPE should not be thrown. + } + } + + @Test(expected = MetaException.class) + public void testAddPartitionSpecWithSharedSDNullLocation() throws Exception { + + createTable(); + PartitionWithoutSD partition = buildPartitionWithoutSD(Lists.newArrayList("2002"), 0); + partition.setRelativePath("year2002"); + String location = null; + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), buildSD(location)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test(expected = MetaException.class) + public void testAddPartitionSpecWithSharedSDEmptyLocation() throws Exception { + + createTable(); + PartitionWithoutSD partition = buildPartitionWithoutSD(Lists.newArrayList("2002"), 0); + partition.setRelativePath("year2002"); + PartitionSpecProxy partitionSpecProxy = buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), buildSD("")); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test(expected = MetaException.class) + public void testAddPartitionSpecWithSharedSDInvalidSD() throws Exception { + + Table table = createTable(); + PartitionWithoutSD partition = buildPartitionWithoutSD(Lists.newArrayList("2002"), 0); + partition.setRelativePath("year2002"); + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation(table.getSd().getLocation() + "/nullLocationTest/"); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), sd); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test + public void testAddPartitionSpecNullLocation() throws Exception { + + Table table = createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE, null); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + + Partition resultPart = + client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList(DEFAULT_YEAR_VALUE)); + Assert.assertEquals(table.getSd().getLocation() + "/year=2017", + resultPart.getSd().getLocation()); + Assert.assertTrue(metaStore.isPathExists(new Path(resultPart.getSd().getLocation()))); + } + + @Test + public void testAddPartitionSpecEmptyLocation() throws Exception { + + Table table = createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE, ""); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + + Partition resultPart = + client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList(DEFAULT_YEAR_VALUE)); + Assert.assertEquals(table.getSd().getLocation() + "/year=2017", + resultPart.getSd().getLocation()); + Assert.assertTrue(metaStore.isPathExists(new Path(resultPart.getSd().getLocation()))); + } + + @Test + public void testAddPartitionSpecEmptyLocationInTableToo() throws Exception { + + Table table = createTable(DB_NAME, TABLE_NAME, null); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE, ""); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + + Partition resultPart = + client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList(DEFAULT_YEAR_VALUE)); + Assert.assertEquals(table.getSd().getLocation() + "/year=2017", + resultPart.getSd().getLocation()); + Assert.assertTrue(metaStore.isPathExists(new Path(resultPart.getSd().getLocation()))); + } + + @Test(expected=MetaException.class) + public void testAddPartitionSpecForView() throws Exception { + + Table table = new TableBuilder() + .setDbName(DB_NAME) + .setTableName(TABLE_NAME) + .setType("VIRTUAL_VIEW") + .addCol("test_id", "int", "test col id") + .addCol("test_value", DEFAULT_COL_TYPE, "test col value") + .addPartCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) + .setLocation(null) + .build(); + client.createTable(table); + + Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test + public void testAddPartitionSpecWithSharedSDNoValue() throws Exception { + + Table table = createTable(); + PartitionWithoutSD partition = new PartitionWithoutSD(); + partition.setRelativePath("addpartspectest"); + String location = table.getSd().getLocation() + "/nullValueTest/"; + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), buildSD(location)); + try { + client.add_partitions_pspec(partitionSpecProxy); + Assert.fail("Exception should have been thrown."); + } catch (NullPointerException | TTransportException e) { + // TODO: NPE should not be thrown. + } + } + + @Test(expected=MetaException.class) + public void testAddPartitionSpecNoValue() throws Exception { + + createTable(); + Partition partition = new PartitionBuilder() + .setDbName(DB_NAME) + .setTableName(TABLE_NAME) + .addCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) + .setLocation(metaStore.getWarehouseRoot() + "/addpartspectest") + .build(); + + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test + public void testAddPartitionSpecNullValue() throws Exception { + + createTable(); + Partition partition = buildPartition(DB_NAME, TABLE_NAME, null); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + try { + client.add_partitions_pspec(partitionSpecProxy); + } catch (NullPointerException e) { + // TODO: NPE should not be thrown + } + } + + @Test + public void testAddPartitionSpecWithSharedSDEmptyValue() throws Exception { + + Table table = createTable(); + PartitionWithoutSD partition = new PartitionWithoutSD(); + partition.setRelativePath("addpartspectest"); + partition.setValues(Lists.newArrayList("")); + String location = table.getSd().getLocation() + "/nullValueTest/"; + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), buildSD(location)); + client.add_partitions_pspec(partitionSpecProxy); + + List partitionNames = client.listPartitionNames(DB_NAME, TABLE_NAME, MAX); + Assert.assertNotNull(partitionNames); + Assert.assertTrue(partitionNames.size() == 1); + Assert.assertEquals("year=__HIVE_DEFAULT_PARTITION__", partitionNames.get(0)); + } + + @Test(expected = MetaException.class) + public void testAddPartitionSpecMoreValues() throws Exception { + + createTable(); + Partition partition = + buildPartition(Lists.newArrayList("2017", "march"), getYearAndMonthPartCols(), 1); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); + client.add_partitions_pspec(partitionSpecProxy); + } + + @Test + public void testAddPartitionSpecWithSharedSDNoRelativePath() throws Exception { + + Table table = createTable(); + PartitionWithoutSD partition1 = buildPartitionWithoutSD(Lists.newArrayList("2007"), 0); + PartitionWithoutSD partition2 = buildPartitionWithoutSD(Lists.newArrayList("2008"), 0); + partition1.setRelativePath(null); + partition2.setRelativePath(null); + String location = table.getSd().getLocation() + "/noRelativePath/"; + PartitionSpecProxy partitionSpecProxy = buildPartitionSpecWithSharedSD( + Lists.newArrayList(partition1, partition2), buildSD(location)); + client.add_partitions_pspec(partitionSpecProxy); + + Partition resultPart1 = client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList("2007")); + Assert.assertEquals(location + "null", resultPart1.getSd().getLocation()); + Assert.assertTrue(metaStore.isPathExists(new Path(resultPart1.getSd().getLocation()))); + + Partition resultPart2 = client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList("2008")); + Assert.assertEquals(location + "null", resultPart2.getSd().getLocation()); + Assert.assertTrue(metaStore.isPathExists(new Path(resultPart2.getSd().getLocation()))); + } + + @Test + public void testAddPartitionSpecOneInvalid() throws Exception { + + createTable(); + Partition partition1 = buildPartition(DB_NAME, TABLE_NAME, "2016"); + Partition partition2 = buildPartition(DB_NAME, TABLE_NAME, "2017"); + Partition partition3 = + buildPartition(Lists.newArrayList("2015", "march"), getYearAndMonthPartCols(), 1); + partition3.getSd().setLocation(metaStore.getWarehouseRoot() + "/addparttest"); + List partitions = Lists.newArrayList(partition1, partition2, partition3); + PartitionSpecProxy partitionSpecProxy = + buildPartitionSpec(DB_NAME, TABLE_NAME, null, partitions); + try { + client.add_partitions_pspec(partitionSpecProxy); + Assert.fail("MetaException should have occurred."); + } catch (MetaException e) { + // This is expected + } + + List parts = client.listPartitions(DB_NAME, TABLE_NAME, MAX); + Assert.assertNotNull(parts); + Assert.assertTrue(parts.isEmpty()); + // TODO: This does not work correctly. None of the partitions is created, but the folder + // for the first two is created. It is because in HiveMetaStore.add_partitions_core when + // going through the partitions, the first two are already put and started in the thread + // pool when the exception occurs in the third one. + // When the exception occurs, we go to the finally part, but the map can be empty + // (it depends on the progress of the other threads) so the folders won't be deleted. + // Assert.assertTrue(metaStore.isPathExists(new Path(partition1.getSd().getLocation()))); + // Assert.assertTrue(metaStore.isPathExists(new Path(partition2.getSd().getLocation()))); + // Assert.assertTrue(metaStore.isPathExists(new Path(partition3.getSd().getLocation()))); + } + // Helper methods private void createDB(String dbName) throws TException { Database db = new DatabaseBuilder().setName(dbName).build(); @@ -1492,4 +2213,112 @@ private void verifyPartitionAttributesDefaultValues(Partition partition, String Assert.assertTrue("Per default the skewedInfo column value location map should be empty.", skewedInfo.getSkewedColValueLocationMaps().isEmpty()); } + + private PartitionWithoutSD buildPartitionWithoutSD(List values, int index) + throws MetaException { + PartitionWithoutSD partition = new PartitionWithoutSD(); + partition.setCreateTime(123456); + partition.setLastAccessTime(123456); + partition.setValues(values); + Map parameters = new HashMap<>(); + parameters.put(DEFAULT_PARAM_KEY + index, DEFAULT_PARAM_VALUE + index); + partition.setParameters(parameters); + partition.setRelativePath("partwithoutsd" + index); + return partition; + } + + private PartitionSpecProxy buildPartitionSpec(String dbName, String tableName, String rootPath, + List partitions) { + + PartitionSpec partitionSpec = new PartitionSpec(); + partitionSpec.setDbName(dbName); + partitionSpec.setRootPath(rootPath); + partitionSpec.setTableName(tableName); + + PartitionListComposingSpec partitionListComposingSpec = new PartitionListComposingSpec(); + partitionListComposingSpec.setPartitions(partitions); + partitionSpec.setPartitionList(partitionListComposingSpec); + + return PartitionSpecProxy.Factory.get(partitionSpec); + } + + private StorageDescriptor buildSD(String location) { + StorageDescriptor sd = new StorageDescriptor(); + sd.setInputFormat("TestInputFormat"); + sd.setOutputFormat("TestOutputFormat"); + sd.setCols(getYearPartCol()); + sd.setCompressed(false); + Map parameters = new HashMap<>(); + parameters.put("testSDParamKey", "testSDParamValue"); + sd.setParameters(parameters); + sd.setLocation(location); + SerDeInfo serdeInfo = new SerDeInfo(); + serdeInfo.setName("sharedSDPartSerde"); + sd.setSerdeInfo(serdeInfo); + return sd; + } + + private PartitionSpecProxy buildPartitionSpecWithSharedSD(List partitions, + StorageDescriptor sd) { + + PartitionSpec partitionSpec = new PartitionSpec(); + partitionSpec.setDbName(DB_NAME); + partitionSpec.setTableName(TABLE_NAME); + PartitionSpecWithSharedSD partitionList = new PartitionSpecWithSharedSD(); + partitionList.setPartitions(partitions); + partitionList.setSd(sd); + partitionSpec.setSharedSDPartitionSpec(partitionList); + return PartitionSpecProxy.Factory.get(partitionSpec); + } + + private PartitionSpecProxy buildPartitionSpec(List partitions, + List partitionsWithoutSD) throws MetaException { + + List partitionSpecs = new ArrayList<>(); + PartitionSpec partitionSpec = new PartitionSpec(); + partitionSpec.setDbName(DB_NAME); + partitionSpec.setTableName(TABLE_NAME); + PartitionListComposingSpec partitionListComposingSpec = new PartitionListComposingSpec(); + partitionListComposingSpec.setPartitions(partitions); + partitionSpec.setPartitionList(partitionListComposingSpec); + + PartitionSpec partitionSpecSharedSD = new PartitionSpec(); + partitionSpecSharedSD.setDbName(DB_NAME); + partitionSpecSharedSD.setTableName(TABLE_NAME); + PartitionSpecWithSharedSD partitionSpecWithSharedSD = new PartitionSpecWithSharedSD(); + partitionSpecWithSharedSD.setPartitions(partitionsWithoutSD); + partitionSpecWithSharedSD + .setSd(buildSD(metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/sharedSDTest/")); + partitionSpecSharedSD.setSharedSDPartitionSpec(partitionSpecWithSharedSD); + + partitionSpecs.add(partitionSpec); + partitionSpecs.add(partitionSpecSharedSD); + return PartitionSpecProxy.Factory.get(partitionSpecs); + } + + private void verifyPartitionSharedSD(Table table, String name, List values, int index) + throws Exception { + + Partition part = client.getPartition(table.getDbName(), table.getTableName(), name); + Assert.assertNotNull(part); + Assert.assertEquals(table.getTableName(), part.getTableName()); + List partValues = part.getValues(); + Assert.assertEquals(values.size(), partValues.size()); + Assert.assertTrue(partValues.containsAll(values)); + Assert.assertEquals(table.getDbName(), part.getDbName()); + Assert.assertEquals(123456, part.getLastAccessTime()); + Assert.assertEquals(DEFAULT_PARAM_VALUE + index, + part.getParameters().get(DEFAULT_PARAM_KEY + index)); + Assert.assertFalse(part.getParameters().keySet().contains(table.getParameters().keySet())); + StorageDescriptor sd = part.getSd(); + Assert.assertNotNull(sd); + Assert.assertEquals("TestInputFormat", sd.getInputFormat()); + Assert.assertEquals("TestOutputFormat", sd.getOutputFormat()); + Assert.assertEquals("sharedSDPartSerde", sd.getSerdeInfo().getName()); + Assert.assertEquals("testSDParamValue", sd.getParameters().get("testSDParamKey")); + Assert.assertEquals( + metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/sharedSDTest/partwithoutsd" + index, + sd.getLocation()); + Assert.assertTrue(metaStore.isPathExists(new Path(sd.getLocation()))); + } }