diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 506bf5dfa8..f12964fa06 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -85,7 +85,13 @@ import static org.apache.hadoop.hive.metastore.Warehouse.makePartName; import static org.apache.hadoop.hive.metastore.Warehouse.makeSpecFromName; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.compareFieldColumns; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getColumnNamesForTable; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getPvals; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.isExternalTable; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.makePartNameMatcher; + /** * todo: This need review re: thread safety. Various places (see callsers of @@ -199,7 +205,7 @@ public void truncateTable(String dbName, String tableName, return deepCopy(table); // Original method used deepCopy(), do the same here. } // Try underlying client - return super.getTable(MetaStoreUtils.getDefaultCatalog(conf), dbname, name, getColStats, engine); + return super.getTable(getDefaultCatalog(conf), dbname, name, getColStats, engine); } // Need to override this one too or dropTable breaks because it doesn't find the table when checks @@ -574,8 +580,8 @@ private void createTempTable(org.apache.hadoop.hive.metastore.api.Table tbl, // Add temp table info to current session Table tTable = new Table(tbl); if (!isVirtualTable) { - StatsSetupConst.setStatsStateForCreateTable(tbl.getParameters(), - org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getColumnNamesForTable(tbl), StatsSetupConst.TRUE); + StatsSetupConst.setStatsStateForCreateTable(tbl.getParameters(), getColumnNamesForTable(tbl), + StatsSetupConst.TRUE); } if (tables == null) { tables = new HashMap(); @@ -782,7 +788,7 @@ private void dropTempTable(org.apache.hadoop.hive.metastore.api.Table table, boo removePartitionedTempTable(table); // Delete table data - if (deleteData && !MetaStoreUtils.isExternalTable(table)) { + if (deleteData && !isExternalTable(table)) { try { boolean ifPurge = false; if (envContext != null){ @@ -1149,7 +1155,7 @@ private Partition getPartition(List partVals) throws MetaException { if (partialPartVals == null || partialPartVals.isEmpty()) { throw new MetaException("Partition partial vals cannot be null or empty"); } - String partNameMatcher = MetaStoreUtils.makePartNameMatcher(tTable, partialPartVals, ".*"); + String partNameMatcher = makePartNameMatcher(tTable, partialPartVals, ".*"); List matchedPartitions = new ArrayList<>(); for (String key : parts.keySet()) { if (key.matches(partNameMatcher)) { @@ -1566,6 +1572,94 @@ public boolean dropPartition(String catName, String dbName, String tableName, St + "supported"); } + @Override + public Partition exchange_partition(Map partitionSpecs, String sourceCatName, + String sourceDbName, String sourceTableName, String destCatName, String destDbName, String destTableName) + throws TException { + org.apache.hadoop.hive.metastore.api.Table sourceTable = getTempTable(sourceDbName, sourceTableName); + org.apache.hadoop.hive.metastore.api.Table destTable = getTempTable(destDbName, destTableName); + if (sourceTable == null || destTable == null) { + return super + .exchange_partition(partitionSpecs, sourceCatName, sourceDbName, sourceTableName, destCatName, destDbName, + destTableName); + } + TempTable sourceTempTable = getPartitionedTempTable(sourceTable); + TempTable destTempTable = getPartitionedTempTable(destTable); + List partitions = + exchangePartitions(partitionSpecs, sourceTable, sourceTempTable, destTable, destTempTable); + if (partitions != null || !partitions.isEmpty()) { + return partitions.get(0); + } + return null; + } + + @Override + public List exchange_partitions(Map partitionSpecs, String sourceCatName, + String sourceDbName, String sourceTableName, String destCatName, String destDbName, String destTableName) + throws TException { + org.apache.hadoop.hive.metastore.api.Table sourceTable = getTempTable(sourceDbName, sourceTableName); + org.apache.hadoop.hive.metastore.api.Table destTable = getTempTable(destDbName, destTableName); + if (sourceTable == null || destTable == null) { + return super + .exchange_partitions(partitionSpecs, sourceCatName, sourceDbName, sourceTableName, destCatName, destDbName, + destTableName); + } + return exchangePartitions(partitionSpecs, sourceTable, getPartitionedTempTable(sourceTable), destTable, + getPartitionedTempTable(destTable)); + } + + private List exchangePartitions(Map partitionSpecs, + org.apache.hadoop.hive.metastore.api.Table sourceTable, TempTable sourceTempTable, + org.apache.hadoop.hive.metastore.api.Table destTable, TempTable destTempTable) throws TException { + if (partitionSpecs == null || partitionSpecs.isEmpty()) { + throw new MetaException("PartitionSpecs cannot be null or empty."); + } + List partitionVals = getPvals(sourceTable.getPartitionKeys(), partitionSpecs); + if (partitionVals.stream().allMatch(String::isEmpty)) { + throw new MetaException("Invalid partition key & values; keys " + + Arrays.toString(sourceTable.getPartitionKeys().toArray()) + ", values " + + Arrays.toString(partitionVals.toArray())); + } + List partitionsToExchange = sourceTempTable + .getPartitionsByPartitionVals(partitionVals); + if (partitionSpecs == null) { + throw new MetaException("The partition specs must be not null."); + } + if (partitionsToExchange.isEmpty()) { + throw new MetaException( + "No partition is found with the values " + partitionSpecs + " for the table " + sourceTable.getTableName()); + } + + boolean sameColumns = compareFieldColumns(sourceTable.getSd().getCols(), destTable.getSd().getCols()); + boolean samePartitions = compareFieldColumns(sourceTable.getPartitionKeys(), destTable.getPartitionKeys()); + if (!(sameColumns && samePartitions)) { + throw new MetaException("The tables have different schemas. Their partitions cannot be exchanged."); + } + // Check if any of the partitions already exists in the destTable + for (Partition partition : partitionsToExchange) { + String partToExchangeName = makePartName(destTable.getPartitionKeys(), partition.getValues()); + if (destTempTable.getPartition(partToExchangeName) != null) { + throw new MetaException( + "The partition " + partToExchangeName + " already exists in the table " + destTable.getTableName()); + } + } + + List result = new ArrayList<>(); + for (Partition partition : partitionsToExchange) { + Partition destPartition = new Partition(partition); + destPartition.setCatName(destTable.getCatName()); + destPartition.setDbName(destTable.getDbName()); + destPartition.setTableName(destTable.getTableName()); + destPartition.getSd().setLocation(""); + destPartition.getSd().setLocation(getPartitionLocation(destTable, destPartition).toString()); + destPartition = destTempTable.addPartition(destPartition); + wh.renameDir(new Path(partition.getSd().getLocation()), new Path(destPartition.getSd().getLocation()), false); + dropPartition(sourceTable.getDbName(), sourceTable.getTableName(), partition.getValues()); + result.add(destPartition); + } + return result; + } + private TempTable getPartitionedTempTable(org.apache.hadoop.hive.metastore.api.Table t) throws MetaException { String qualifiedTableName = Warehouse. getQualifiedName(t.getDbName().toLowerCase(), t.getTableName().toLowerCase()); diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java new file mode 100644 index 0000000000..956346a865 --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestSessionHiveMetastoreClientExchangePartitionsTempTable.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.metadata; + +import com.google.common.collect.Lists; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.TestExchangePartitions; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Test class for exchange partitions related methods on temporary tables. + */ +@RunWith(Parameterized.class) +@Category(MetastoreCheckinTest.class) +public class TestSessionHiveMetastoreClientExchangePartitionsTempTable extends TestExchangePartitions { + + private HiveConf conf; + private Warehouse wh; + + public TestSessionHiveMetastoreClientExchangePartitionsTempTable(String name, AbstractMetaStoreService metaStore) { + super(name, metaStore); + } + + @Before + @Override + public void setUp() throws Exception { + initHiveConf(); + wh = new Warehouse(conf); + SessionState.start(conf); + setClient(Hive.get(conf).getMSC()); + getClient().dropDatabase(DB_NAME, true, true, true); + getMetaStore().cleanWarehouseDirs(); + createTestTables(); + } + + private void initHiveConf() throws Exception{ + conf = Hive.get().getConf(); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); + } + + @Override + protected Table createTable(String dbName, String tableName, List partCols, + List cols, String location) throws Exception { + new TableBuilder() + .setDbName(dbName) + .setTableName(tableName) + .setCols(cols) + .setPartCols(partCols) + .setLocation(location) + .setTemporary(true) + .create(getClient(), getMetaStore().getConf()); + return getClient().getTable(dbName, tableName); + } + + @Override + @Test(expected = MetaException.class) + public void testExchangePartitionsNonExistingPartLocation() throws Exception { + Map partitionSpecs = getPartitionSpec(getPartitions()[1]); + getMetaStore().cleanWarehouseDirs(); + cleanTempTableDir(getSourceTable()); + cleanTempTableDir(getDestTable()); + getClient().exchange_partitions(partitionSpecs, getSourceTable().getDbName(), + getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName()); + } + + @Override + @Test + public void testExchangePartitionsCustomTableAndPartLocation() throws Exception { + Table source = createTable(DB_NAME, "test_source_table_cust_loc", + getYearMonthAndDayPartCols(), getMetaStore().getWarehouseRoot() + "/sourceTable"); + Table dest = createTable(DB_NAME, "test_dest_table_cust_loc", getYearMonthAndDayPartCols(), + getMetaStore().getWarehouseRoot() + "/destTable"); + org.apache.hadoop.hive.metastore.api.Partition[] parts = new Partition[2]; + parts[0] = createPartition(source, Lists.newArrayList("2019", "may", "11"), + source.getSd().getLocation() + "/2019m11"); + parts[1] = createPartition(source, Lists.newArrayList("2019", "july", "23"), + source.getSd().getLocation() + "/2019j23"); + + Map partitionSpecs = getPartitionSpec(parts[1]); + getClient().exchange_partitions(partitionSpecs, source.getDbName(), + source.getTableName(), dest.getDbName(), dest.getTableName()); + + checkRemainingPartitions(source, dest, Lists.newArrayList(parts[0])); + List destTablePartitions = getClient().listPartitions(dest.getDbName(), dest.getTableName(), (short) -1); + Assert.assertEquals(1, destTablePartitions.size()); + checkExchangedPartitions(source, dest, Lists.newArrayList(parts[1])); + } + + @Override + @Test + public void testExchangePartitionsCustomPartLocation() throws Exception { + Table source = createTable(DB_NAME, "test_source_table", getYearMonthAndDayPartCols(), null); + Table dest = createTable(DB_NAME, "test_dest_table", getYearMonthAndDayPartCols(), null); + Partition[] parts = new Partition[2]; + parts[0] = createPartition(source, Lists.newArrayList("2019", "march", "15"), + source.getSd().getLocation() + "/2019m15"); + parts[1] = createPartition(source, Lists.newArrayList("2019", "march", "22"), + source.getSd().getLocation() + "/2019m22"); + + Map partitionSpecs = getPartitionSpec(parts[1]); + getClient().exchange_partitions(partitionSpecs, source.getDbName(), + source.getTableName(), dest.getDbName(), dest.getTableName()); + + checkRemainingPartitions(source, dest, Lists.newArrayList(parts[0])); + List destTablePartitions = getClient().listPartitions(dest.getDbName(), dest.getTableName(), (short) -1); + Assert.assertEquals(1, destTablePartitions.size()); + checkExchangedPartitions(source, dest, Lists.newArrayList(parts[1])); + } + + @Override + @Test + public void testExchangePartitionsOnlyMonthSetInPartSpec() throws Exception { + Map partitionSpecs = new HashMap<>(); + partitionSpecs.put(YEAR_COL_NAME, ""); + partitionSpecs.put(MONTH_COL_NAME, "march"); + partitionSpecs.put(DAY_COL_NAME, ""); + + getClient().exchange_partitions(partitionSpecs, getSourceTable().getDbName(), + getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName()); + checkRemainingPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[2], + getPartitions()[3], getPartitions()[4])); + List exchangedPartitions = getClient().listPartitions(getDestTable().getDbName(), + getDestTable().getTableName(), MAX); + Assert.assertEquals(2, exchangedPartitions.size()); + checkExchangedPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[0], + getPartitions()[1])); + } + + @Override + @Test + public void testExchangePartitionsYearAndDaySetInPartSpec() throws Exception { + Map partitionSpecs = new HashMap<>(); + partitionSpecs.put(YEAR_COL_NAME, "2017"); + partitionSpecs.put(MONTH_COL_NAME, ""); + partitionSpecs.put(DAY_COL_NAME, "22"); + getClient().exchange_partitions(partitionSpecs, getSourceTable().getDbName(), + getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName()); + checkRemainingPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[0], + getPartitions()[2], getPartitions()[3], getPartitions()[4])); + List exchangedPartitions = getClient().listPartitions(getDestTable().getDbName(), + getDestTable().getTableName(), MAX); + Assert.assertEquals(1, exchangedPartitions.size()); + checkExchangedPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[1])); + } + + @Override + @Test + public void testExchangePartition() throws Exception { + Map partitionSpecs = getPartitionSpec(getPartitions()[1]); + Partition exchangedPartition = + getClient().exchange_partition(partitionSpecs, getSourceTable().getDbName(), + getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName()); + Assert.assertNotNull(exchangedPartition); + checkExchangedPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[1])); + checkRemainingPartitions(getSourceTable(), getDestTable(), + Lists.newArrayList(getPartitions()[0], getPartitions()[2], getPartitions()[3], getPartitions()[4])); + } + + @Override + @Test + public void testExchangePartitionYearSet() throws Exception { + Map partitionSpecs = getPartitionSpec(Lists.newArrayList("2017", "", "")); + Partition exchangedPartition = + getClient().exchange_partition(partitionSpecs, getSourceTable().getDbName(), + getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName()); + Assert.assertNotNull(exchangedPartition); + checkExchangedPartitions(getSourceTable(), getDestTable(), + Lists.newArrayList(getPartitions()[0], getPartitions()[1], getPartitions()[2], getPartitions()[3])); + checkRemainingPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[4])); + } + + @Override + @Test + public void testExchangePartitionCustomTableAndPartLocation() throws Exception { + Table source = createTable(DB_NAME, "test_source_table_cust_loc", + getYearMonthAndDayPartCols(), getMetaStore().getWarehouseRoot() + "/sourceTable"); + Table dest = createTable(DB_NAME, "test_dest_table_cust_loc", getYearMonthAndDayPartCols(), + getMetaStore().getWarehouseRoot() + "/destTable"); + Partition[] parts = new Partition[2]; + parts[0] = createPartition(source, Lists.newArrayList("2019", "may", "11"), + source.getSd().getLocation() + "/2019m11"); + parts[1] = createPartition(source, Lists.newArrayList("2019", "july", "23"), + source.getSd().getLocation() + "/2019j23"); + + Map partitionSpecs = getPartitionSpec(parts[1]); + getClient().exchange_partition(partitionSpecs, source.getDbName(), + source.getTableName(), dest.getDbName(), dest.getTableName()); + + checkRemainingPartitions(source, dest, Lists.newArrayList(parts[0])); + List destTablePartitions = + getClient().listPartitions(dest.getDbName(), dest.getTableName(), (short) -1); + Assert.assertEquals(1, destTablePartitions.size()); + checkExchangedPartitions(source, dest, Lists.newArrayList(parts[1])); + } + + @Override + @Test + public void testExchangePartitionCustomPartLocation() throws Exception { + Table source = createTable(DB_NAME, "test_source_table", getYearMonthAndDayPartCols(), null); + Table dest = createTable(DB_NAME, "test_dest_table", getYearMonthAndDayPartCols(), null); + Partition[] parts = new Partition[2]; + parts[0] = createPartition(source, Lists.newArrayList("2019", "march", "15"), + source.getSd().getLocation() + "/2019m15"); + parts[1] = createPartition(source, Lists.newArrayList("2019", "march", "22"), + source.getSd().getLocation() + "/2019m22"); + + Map partitionSpecs = getPartitionSpec(parts[1]); + getClient().exchange_partition(partitionSpecs, source.getDbName(), + source.getTableName(), dest.getDbName(), dest.getTableName()); + checkRemainingPartitions(source, dest, Lists.newArrayList(parts[0])); + List destTablePartitions = + getClient().listPartitions(dest.getDbName(), dest.getTableName(), (short) -1); + Assert.assertEquals(1, destTablePartitions.size()); + checkExchangedPartitions(source, dest, Lists.newArrayList(parts[1])); + } + + @Override + @Test(expected = MetaException.class) + public void testExchangePartitionNonExistingPartLocation() throws Exception { + Map partitionSpecs = getPartitionSpec(getPartitions()[1]); + cleanTempTableDir(getSourceTable()); + cleanTempTableDir(getDestTable()); + getClient().exchange_partition(partitionSpecs, getSourceTable().getDbName(), + getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName()); + } + + @Override + @Test + public void testExchangePartitionOnlyMonthSetInPartSpec() throws Exception { + Map partitionSpecs = new HashMap<>(); + partitionSpecs.put(YEAR_COL_NAME, ""); + partitionSpecs.put(MONTH_COL_NAME, "march"); + partitionSpecs.put(DAY_COL_NAME, ""); + getClient().exchange_partitions(partitionSpecs, getSourceTable().getDbName(), + getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName()); + checkRemainingPartitions(getSourceTable(), getDestTable(), + Lists.newArrayList(getPartitions()[2], getPartitions()[3], getPartitions()[4])); + List exchangedPartitions = getClient().listPartitions(getDestTable().getDbName(), + getDestTable().getTableName(), MAX); + Assert.assertEquals(2, exchangedPartitions.size()); + checkExchangedPartitions(getSourceTable(), getDestTable(), + Lists.newArrayList(getPartitions()[0], getPartitions()[1])); + } + + @Override + @Test + public void testExchangePartitionYearAndDaySetInPartSpec() throws Exception { + Map partitionSpecs = new HashMap<>(); + partitionSpecs.put(YEAR_COL_NAME, "2017"); + partitionSpecs.put(MONTH_COL_NAME, ""); + partitionSpecs.put(DAY_COL_NAME, "22"); + getClient().exchange_partition(partitionSpecs, getSourceTable().getDbName(), + getSourceTable().getTableName(), getDestTable().getDbName(), getDestTable().getTableName()); + checkRemainingPartitions(getSourceTable(), getDestTable(), + Lists.newArrayList(getPartitions()[0], getPartitions()[2], getPartitions()[3], getPartitions()[4])); + List exchangedPartitions = getClient().listPartitions(getDestTable().getDbName(), + getDestTable().getTableName(), MAX); + Assert.assertEquals(1, exchangedPartitions.size()); + checkExchangedPartitions(getSourceTable(), getDestTable(), Lists.newArrayList(getPartitions()[1])); + } + + private void cleanTempTableDir(Table table) throws MetaException { + wh.deleteDir(new Path(table.getSd().getLocation()), true, false, false); + } + +} diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java index 1a2b7e4f9f..4823f60978 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestExchangePartitions.java @@ -56,16 +56,16 @@ private AbstractMetaStoreService metaStore; private IMetaStoreClient client; - private static final String DB_NAME = "test_partition_db"; + protected static final String DB_NAME = "test_partition_db"; private static final String STRING_COL_TYPE = "string"; private static final String INT_COL_TYPE = "int"; - private static final String YEAR_COL_NAME = "year"; - private static final String MONTH_COL_NAME = "month"; - private static final String DAY_COL_NAME = "day"; - private static final short MAX = -1; - private static Table sourceTable; - private static Table destTable; - private static Partition[] partitions; + protected static final String YEAR_COL_NAME = "year"; + protected static final String MONTH_COL_NAME = "month"; + protected static final String DAY_COL_NAME = "day"; + protected static final short MAX = -1; + private Table sourceTable; + private Table destTable; + private Partition[] partitions; public TestExchangePartitions(String name, AbstractMetaStoreService metaStore) { this.metaStore = metaStore; @@ -79,10 +79,7 @@ public void setUp() throws Exception { // Clean up the database client.dropDatabase(DB_NAME, true, true, true); metaStore.cleanWarehouseDirs(); - createDB(DB_NAME); - sourceTable = createSourceTable(); - destTable = createDestTable(); - partitions = createTestPartitions(); + createTestTables(); } @After @@ -100,6 +97,30 @@ public void tearDown() throws Exception { } } + protected IMetaStoreClient getClient() { + return client; + } + + protected void setClient(IMetaStoreClient client) { + this.client = client; + } + + protected AbstractMetaStoreService getMetaStore() { + return metaStore; + } + + protected Table getSourceTable() { + return sourceTable; + } + + protected Table getDestTable() { + return destTable; + } + + protected Partition[] getPartitions() { + return partitions; + } + // Tests for the List exchange_partitions(Map partitionSpecs, String // sourceDb, String sourceTable, String destdb, String destTableName) method @@ -1164,6 +1185,13 @@ public void testExchangePartitionNoPartExistsYearAndMonthSet() throws Exception } // Helper methods + protected void createTestTables() throws Exception { + createDB(DB_NAME); + sourceTable = createSourceTable(); + destTable = createDestTable(); + partitions = createTestPartitions(); + } + private void createDB(String dbName) throws TException { new DatabaseBuilder() .setName(dbName) @@ -1178,7 +1206,7 @@ private Table createDestTable() throws Exception { return createTable(DB_NAME, "test_part_exch_table_dest", getYearMonthAndDayPartCols(), null); } - private Table createTable(String dbName, String tableName, List partCols, + protected Table createTable(String dbName, String tableName, List partCols, String location) throws Exception { List cols = new ArrayList<>(); cols.add(new FieldSchema("test_id", INT_COL_TYPE, "test col id")); @@ -1186,7 +1214,7 @@ private Table createTable(String dbName, String tableName, List par return createTable(dbName, tableName, partCols, cols, location); } - private Table createTable(String dbName, String tableName, List partCols, + protected Table createTable(String dbName, String tableName, List partCols, List cols, String location) throws Exception { new TableBuilder() .setDbName(dbName) @@ -1226,7 +1254,7 @@ private Table createTable(String dbName, String tableName, List par return parts; } - private Partition createPartition(Table table, List values, String location) + protected Partition createPartition(Table table, List values, String location) throws Exception { Partition partition = buildPartition(table, values, location); client.add_partition(partition); @@ -1249,7 +1277,7 @@ private Partition buildPartition(Table table, List values, String locati return partition; } - private void checkExchangedPartitions(Table sourceTable, Table destTable, + protected void checkExchangedPartitions(Table sourceTable, Table destTable, List partitions) throws Exception { for (Partition partition : partitions) { @@ -1286,7 +1314,7 @@ private void checkExchangedPartitions(Table sourceTable, Table destTable, } } - private void checkRemainingPartitions(Table sourceTable, Table destTable, + protected void checkRemainingPartitions(Table sourceTable, Table destTable, List partitions) throws Exception { for (Partition partition : partitions) { @@ -1312,11 +1340,11 @@ private void checkRemainingPartitions(Table sourceTable, Table destTable, } } - private static Map getPartitionSpec(Partition partition) { + protected static Map getPartitionSpec(Partition partition) { return getPartitionSpec(partition.getValues()); } - private static Map getPartitionSpec(List values) { + protected static Map getPartitionSpec(List values) { Map partitionSpecs = new HashMap<>(); List partCols = getYearMonthAndDayPartCols(); for (int i = 0; i < partCols.size(); i++) { @@ -1327,7 +1355,7 @@ private void checkRemainingPartitions(Table sourceTable, Table destTable, return partitionSpecs; } - private static List getYearMonthAndDayPartCols() { + protected static List getYearMonthAndDayPartCols() { List cols = new ArrayList<>(); cols.add(new FieldSchema(YEAR_COL_NAME, STRING_COL_TYPE, "year part col")); cols.add(new FieldSchema(MONTH_COL_NAME, STRING_COL_TYPE, "month part col"));