diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java new file mode 100644 index 0000000000..3aaad22711 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestTransactionalValidationListener.java @@ -0,0 +1,127 @@ +package org.apache.hadoop.hive.metastore; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.api.Catalog; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.MetaStoreClientTest; +import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(Parameterized.class) +public class TestTransactionalValidationListener extends MetaStoreClientTest { + + private AbstractMetaStoreService metaStore; + private IMetaStoreClient client; + private boolean createdCatalogs = false; + + @BeforeClass + public static void startMetaStores() { + Map msConf = new HashMap(); + + // Enable TransactionalValidationListener + create.as.acid + Map extraConf = new HashMap<>(); + extraConf.put("metastore.create.as.acid", "true"); + extraConf.put("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); + extraConf.put("hive.support.concurrency", "true"); + startMetaStores(msConf, extraConf); + } + + @Before + public void setUp() throws Exception { + // Get new client + client = metaStore.getClient(); + if (!createdCatalogs) { + createCatalogs(); + createdCatalogs = true; + } + } + + @After + public void tearDown() throws Exception { + try { + if (client != null) { + client.close(); + } + } finally { + client = null; + } + } + + public TestTransactionalValidationListener(String name, AbstractMetaStoreService metaStore) throws Exception { + this.metaStore = metaStore; + } + + private void createCatalogs() throws Exception { + String[] catNames = {"spark", "myapp"}; + String[] location = {MetaStoreTestUtils.getTestWarehouseDir("spark"), + MetaStoreTestUtils.getTestWarehouseDir("myapp")}; + + for (int i = 0; i < catNames.length; i++) { + Catalog cat = new CatalogBuilder() + .setName(catNames[i]) + .setLocation(location[i]) + .build(); + client.createCatalog(cat); + File dir = new File(cat.getLocationUri()); + Assert.assertTrue(dir.exists() && dir.isDirectory()); + } + } + + private Table createOrcTable(String catalog) throws Exception { + Table table = new Table(); + StorageDescriptor sd = new StorageDescriptor(); + List cols = new ArrayList<>(); + + table.setDbName("default"); + table.setTableName("test_table"); + cols.add(new FieldSchema("column_name", "int", null)); + sd.setCols(cols); + sd.setSerdeInfo(new SerDeInfo()); + sd.setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"); + sd.setOutputFormat("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"); + table.setSd(sd); + table.setCatName(catalog); + table.setTableType("MANAGED_TABLE"); + + client.createTable(table); + Table createdTable = client.getTable(catalog, table.getDbName(), table.getTableName()); + return createdTable; + } + + @Test + public void testCreateAsAcid() throws Exception { + // Table created in hive catalog should have been automatically set to transactional + Table createdTable = createOrcTable("hive"); + assertTrue(AcidUtils.isTransactionalTable(createdTable)); + + // Non-hive catalogs should not be transactional + createdTable = createOrcTable("spark"); + assertFalse(AcidUtils.isTransactionalTable(createdTable)); + + createdTable = createOrcTable("myapp"); + assertFalse(AcidUtils.isTransactionalTable(createdTable)); + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index 56da1151cc..7ef82c2aa9 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -53,8 +53,11 @@ public static final String DEFAULT_TRANSACTIONAL_PROPERTY = "default"; public static final String INSERTONLY_TRANSACTIONAL_PROPERTY = "insert_only"; + private final Set supportedCatalogs = new HashSet(); + TransactionalValidationListener(Configuration conf) { super(conf); + supportedCatalogs.add("hive"); } @Override @@ -73,11 +76,21 @@ public void onEvent(PreEventContext context) throws MetaException, NoSuchObjectE } private void handle(PreAlterTableEvent context) throws MetaException { - handleAlterTableTransactionalProp(context); + if (supportedCatalogs.contains(getTableCatalog(context.getNewTable()))) { + handleAlterTableTransactionalProp(context); + } } private void handle(PreCreateTableEvent context) throws MetaException { - handleCreateTableTransactionalProp(context); + if (supportedCatalogs.contains(getTableCatalog(context.getTable()))) { + handleCreateTableTransactionalProp(context); + } + } + + private String getTableCatalog(Table table) { + String catName = table.isSetCatName() ? table.getCatName() : + MetaStoreUtils.getDefaultCatalog(getConf()); + return catName.toLowerCase(); } /** @@ -230,7 +243,8 @@ private void makeAcid(Table newTable) throws MetaException { newTable.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)); return; } - Configuration conf = MetastoreConf.newMetastoreConf(); + + Configuration conf = getConf(); boolean makeAcid = //no point making an acid table if these other props are not set since it will just throw //exceptions when someone tries to use the table. @@ -437,8 +451,7 @@ private void validateTableStructure(IHMSHandler hmsHandler, Table table) try { Warehouse wh = hmsHandler.getWh(); if (table.getSd().getLocation() == null || table.getSd().getLocation().isEmpty()) { - String catName = table.isSetCatName() ? table.getCatName() : - MetaStoreUtils.getDefaultCatalog(getConf()); + String catName = getTableCatalog(table); tablePath = wh.getDefaultTablePath(hmsHandler.getMS().getDatabase( catName, table.getDbName()), table.getTableName()); } else { diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java index a0e9d32546..dc48fa8308 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java @@ -67,7 +67,7 @@ public static void startMetaStores() { * @param msConf Specific MetaStore configuration values * @param extraConf Specific other configuration values */ - static void startMetaStores(Map msConf, + public static void startMetaStores(Map msConf, Map extraConf) { for(AbstractMetaStoreService metaStoreService : metaStoreServices) { try {