diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 06061c0..c236a9a 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -23,6 +23,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -68,6 +69,7 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; @@ -76,6 +78,7 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.util.StringUtils; import org.apache.thrift.TException; +import org.junit.Assert; import org.junit.Test; import com.google.common.collect.Lists; @@ -2740,6 +2743,69 @@ private void createTable(String dbName, String tableName) createTable(dbName, tableName, null, null, null, sd, 0); } + @Test + public void testTransactionalValidation() throws Throwable { + String tblName = "acidTable"; + String owner = "acid"; + Map fields = new HashMap(); + fields.put("name", serdeConstants.STRING_TYPE_NAME); + fields.put("income", serdeConstants.INT_TYPE_NAME); + + Type typ1 = createType("Person", fields); + + + Map params = new HashMap(); + params.put("transactional", ""); + + Map serdParams = new HashMap(); + serdParams.put(serdeConstants.SERIALIZATION_FORMAT, "1"); + StorageDescriptor sd = createStorageDescriptor(tblName, typ1.getFields(), params, serdParams); + + try { + Table t = createTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName, owner, params, null, sd, 0); + Assert.assertTrue("Expected exception", false); + } + catch(MetaException e) { + Assert.assertEquals("'transactional' property of TBLPROPERTIES may only have value 'true'", e.getMessage()); + } + try { + params.clear(); + params.put("transactional", "foobar"); + Table t = createTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName, owner, params, null, sd, 0); + Assert.assertTrue("Expected exception", false); + } + catch(MetaException e) { + Assert.assertEquals("'transactional' property of TBLPROPERTIES may only have value 'true'", e.getMessage()); + } + params.clear(); + params.put("Transactional", "True"); + Table t = createTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName, owner, params, null, sd, 0); + Assert.assertTrue("", "true".equals(t.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL))); + + try { + params.clear(); + params.put("transactional", "false"); + t = new Table(); + t.setParameters(params); + client.alter_table(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName, t); + Assert.assertTrue("Expected exception", false); + } + catch(MetaException e) { + Assert.assertEquals("TBLPROPERTIES with 'transactional'='true' cannot be unset", e.getMessage()); + } + params.clear(); + t = createTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName + "1", owner + "1", params, null, sd, 0); + + + tblName += "2"; + t = createTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName, owner, params, null, sd, 0); + params.clear(); + params.put("transactional", "true"); + t.setParameters(params); + t.setPartitionKeys(Collections.EMPTY_LIST); + client.alter_table(MetaStoreUtils.DEFAULT_DATABASE_NAME, tblName, t); + + } private Table createTable(String dbName, String tblName, String owner, Map tableParams, Map partitionKeys, StorageDescriptor sd, int lastAccessTime) throws Exception { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index ccb4c98..3367ba3 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -208,7 +208,6 @@ import javax.jdo.JDOException; import java.io.IOException; -import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -487,6 +486,7 @@ public void init() throws MetaException { preListeners = MetaStoreUtils.getMetaStoreListeners(MetaStorePreEventListener.class, hiveConf, hiveConf.getVar(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS)); + preListeners.add(0, new MetaStoreValidationListener(hiveConf)); listeners = MetaStoreUtils.getMetaStoreListeners(MetaStoreEventListener.class, hiveConf, hiveConf.getVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS)); listeners.add(new SessionPropertiesListener(hiveConf)); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreValidationListener.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreValidationListener.java new file mode 100644 index 0000000..4b93b97 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreValidationListener.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; +import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; +import org.apache.hadoop.hive.metastore.events.PreEventContext; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +final class MetaStoreValidationListener extends MetaStorePreEventListener { + MetaStoreValidationListener(Configuration conf) { + super(conf); + } + + public void onEvent(PreEventContext context) throws MetaException, NoSuchObjectException, InvalidOperationException { + switch (context.getEventType()) { + case CREATE_TABLE: + handle((PreCreateTableEvent) context); + break; + case ALTER_TABLE: + handle((PreAlterTableEvent) context); + default: + //no validation required.. + } + } + + private void handle(PreAlterTableEvent context) throws MetaException { + handleAlterTableTransactionalProp(context); + } + + private void handle(PreCreateTableEvent context) throws MetaException { + handleCreateTableTransactionalProp(context); + } + + /** + * once a table is marked transactional, you cannot go back. Enforce this. + */ + private void handleAlterTableTransactionalProp(PreAlterTableEvent context) throws MetaException { + Table newTable = context.getNewTable(); + Map parameters = newTable.getParameters(); + if (parameters == null || parameters.isEmpty()) { + return; + } + Set keys = new HashSet<>(parameters.keySet()); + String transactionalValue = null; + boolean transactionalValuePresent = false; + for(String key : keys) { + if(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) { + transactionalValuePresent = true; + transactionalValue = parameters.get(key); + parameters.remove(key); + } + } + if(transactionalValuePresent) { + //normalize prop name + parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, transactionalValue); + } + if("true".equalsIgnoreCase(transactionalValue)) { + return; + } + Table oldTable = context.getOldTable(); + String oldTransactionalValue = null; + for (String key : oldTable.getParameters().keySet()) { + if (hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) { + oldTransactionalValue = oldTable.getParameters().get(key); + } + } + if(oldTransactionalValue == null ? transactionalValue == null : oldTransactionalValue.equalsIgnoreCase(transactionalValue)) { + //this covers backward compat cases where this prop may have been set already + return; + } + //if here, there is attempt to set transactional to something other than 'true' and NOT the same value it was before + throw new MetaException("TBLPROPERTIES with 'transactional'='true' cannot be unset"); + } + /** + * normalize case and make sure 'true' is the only value it can be set to (if set at all) + */ + private void handleCreateTableTransactionalProp(PreCreateTableEvent context) throws MetaException { + Table newTable = context.getTable(); + Map parameters = newTable.getParameters(); + if (parameters == null || parameters.isEmpty()) { + return; + } + String transactionalValue = null; + boolean transactionalPropFound = false; + Set keys = new HashSet<>(parameters.keySet()); + for(String key : keys) { + if(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) { + transactionalPropFound = true; + transactionalValue = parameters.get(key); + parameters.remove(key); + } + } + if ("false".equalsIgnoreCase(transactionalValue)) { + //just drop transactional=false. For backward compatibility in case someone has scripts + //with transactional=false + return; + } + if("true".equalsIgnoreCase(transactionalValue)) { + //normalize prop name + parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, Boolean.TRUE.toString()); + return; + } + if(transactionalPropFound) { + throw new MetaException("'transactional' property of TBLPROPERTIES may only have value 'true'"); + } + } +} \ No newline at end of file