diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 5bb1985..ee87cdd 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -77,7 +77,9 @@ import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MNotificationLog; import org.apache.hadoop.hive.metastore.model.MNotificationNextId; +import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics; +import org.apache.hadoop.hive.metastore.model.MPartitionPrivilege; import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics; import org.apache.hadoop.hive.metastore.model.MWMResourcePlan; import org.apache.hadoop.hive.metastore.parser.ExpressionTree; @@ -235,6 +237,8 @@ private boolean ensureDbInit() { initQueries.add(pm.newQuery(MNotificationNextId.class, "nextEventId < -1")); initQueries.add(pm.newQuery(MWMResourcePlan.class, "name == ''")); initQueries.add(pm.newQuery(MCreationMetadata.class, "dbName == ''")); + initQueries.add(pm.newQuery(MPartitionPrivilege.class, "principalName == ''")); + initQueries.add(pm.newQuery(MPartitionColumnPrivilege.class, "principalName == ''")); Query q; while ((q = initQueries.peekFirst()) != null) { q.execute(); diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index b15d89d..9b2622f 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -2555,10 +2555,17 @@ public boolean dropPartition(String catName, String dbName, String tableName, @Override public void dropPartitions(String catName, String dbName, String tblName, List partNames) throws MetaException, NoSuchObjectException { + dropPartitionsInternal(catName, dbName, tblName, partNames, true, true); + } + + @VisibleForTesting + protected void dropPartitionsInternal(String catName, String dbName, String tblName, + List partNames, boolean allowSql, boolean allowJdo) + throws MetaException, NoSuchObjectException { if (CollectionUtils.isEmpty(partNames)) { return; } - new GetListHelper(catName, dbName, tblName, true, true) { + new GetListHelper(catName, dbName, tblName, allowSql, allowJdo) { @Override protected List getSqlResult(GetHelper> ctx) throws MetaException { directSql.dropPartitionsViaSqlFilter(catName, dbName, tblName, partNames); diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index 9912213..7eb781a 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -45,6 +45,8 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.messaging.EventMessage; import org.apache.hadoop.hive.metastore.metrics.Metrics; @@ -62,6 +64,11 @@ import org.slf4j.LoggerFactory; import javax.jdo.Query; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -340,6 +347,117 @@ public void testPartitionOps() throws MetaException, InvalidObjectException, } /** + * Checks if the JDO cache is able to handle directSQL partition drops + * @throws MetaException + * @throws InvalidObjectException + * @throws NoSuchObjectException + * @throws SQLException + */ + @Test + public void testDirectSQLDropParitionsCache() throws MetaException, InvalidObjectException, + NoSuchObjectException, SQLException { + createPartitionedTable(); + // query the partitions with JDO + Deadline.startTimer("getPartition"); + List partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1, + 10, false, true); + Assert.assertEquals(3, partitions.size()); + + // drop partitions with directSql + objectStore.dropPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1, + Arrays.asList("test_part_col=a0", "test_part_col=a1"), true, false); + + // query the partitions with JDO, checking the cache is not causing any problem + partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1, + 10, false, true); + Assert.assertEquals(1, partitions.size()); + } + + /** + * Checks if the directSQL partition drop removes every connected data from the RDBMS tables. + * @throws MetaException + * @throws InvalidObjectException + * @throws NoSuchObjectException + * @throws SQLException + */ + @Test + public void testDirectSQLDropParitionsCleanup() throws MetaException, InvalidObjectException, + NoSuchObjectException, SQLException { + + createPartitionedTable(); + + // drop the partitions + Deadline.startTimer("dropPartitions"); + objectStore.dropPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1, + Arrays.asList("test_part_col=a0", "test_part_col=a1", "test_part_col=a2"), true, false); + + // Check, if every data is dropped connected to the partitions + checkBackendTableSize("PARTITIONS", 0); + checkBackendTableSize("PART_PRIVS", 0); + checkBackendTableSize("PART_COL_PRIVS", 0); + checkBackendTableSize("PART_COL_STATS", 0); + checkBackendTableSize("PARTITION_PARAMS", 0); + checkBackendTableSize("PARTITION_KEY_VALS", 0); + checkBackendTableSize("SD_PARAMS", 0); + checkBackendTableSize("BUCKETING_COLS", 0); + checkBackendTableSize("SKEWED_COL_NAMES", 0); + checkBackendTableSize("SDS", 1); // Table has an SDS + checkBackendTableSize("SORT_COLS", 0); + checkBackendTableSize("SERDE_PARAMS", 0); + checkBackendTableSize("SERDES", 1); // Table has a serde + } + + /** + * Creates DB1 database, TABLE1 table with 3 partitions + * @throws MetaException + * @throws InvalidObjectException + */ + private void createPartitionedTable() throws MetaException, InvalidObjectException { + Database db1 = new DatabaseBuilder() + .setName(DB1) + .setDescription("description") + .setLocation("locationurl") + .build(conf); + objectStore.createDatabase(db1); + Table tbl1 = + new TableBuilder() + .setDbName(DB1) + .setTableName(TABLE1) + .addCol("test_col1", "int") + .addCol("test_col2", "int") + .addPartCol("test_part_col", "int") + .build(conf); + objectStore.createTable(tbl1); + + // Create partitions for the partitioned table + for(int i=0; i < 3; i++) { + Partition part = new PartitionBuilder() + .inTable(tbl1) + .addValue("a" + i) + .build(conf); + objectStore.addPartition(part); + } + } + + /** + * Checks if the HMS backend db row number is as expected. If they are not, an + * {@link AssertionError} is thrown. + * @param tableName The table in which we count the rows + * @param size The expected row number + * @throws SQLException If there is a problem connecting to / querying the backend DB + */ + private void checkBackendTableSize(String tableName, int size) throws SQLException { + String connectionStr = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY); + Connection conn = DriverManager.getConnection(connectionStr); + Statement stmt = conn.createStatement(); + + ResultSet rs = stmt.executeQuery("SELECT COUNT(1) FROM " + tableName); + rs.next(); + Assert.assertEquals(tableName + " table should contain " + size + " rows", size, + rs.getLong(1)); + } + + /** * Test master keys operation */ @Test