commit 753eb9c1a025f0ba1e1e063debca5280e53298b6 Author: Janaki Lahorani Date: Fri Nov 3 13:00:59 2017 -0700 HIVE 17942: HiveAlterHandler will now use configuration from HMS Handler so that any changes to the configuration is picked up. Change-Id: I8b480f30d87b7c2f3db7ce9065692feb996015c3 diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreAlterColumnPar.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreAlterColumnPar.java new file mode 100644 index 0000000000000000000000000000000000000000..810c888610292b4fb01e9505e54385fb77069484 --- /dev/null +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreAlterColumnPar.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.HashMap; + +/** + * Test that set/unset of metaconf:hive.metastore.disallow.incompatible.col.type.changes is local + * to the session + */ +public class TestHiveMetaStoreAlterColumnPar { + + public static MiniHS2 miniHS2 = null; + + @BeforeClass + public static void startServices() throws Exception { + HiveConf hiveConf = new HiveConf(); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS, 2); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS, 2); + hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + + miniHS2 = new MiniHS2.Builder().withMiniMR().withRemoteMetastore().withConf(hiveConf).build(); + + miniHS2.start(new HashMap()); + + Connection hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "hive", "hive"); + Statement stmt = hs2Conn.createStatement(); + + // create table + stmt.execute("drop table if exists t1"); + stmt.execute("create table t1 (c1 int)"); + + stmt.close(); + hs2Conn.close(); + } + + @AfterClass + public static void stopServices() throws Exception { + if (miniHS2 != null && miniHS2.isStarted()) { + miniHS2.stop(); + } + } + + @Test + public void testAlterColumn() throws Exception { + assertTrue("Test setup failed. MiniHS2 is not initialized", + miniHS2 != null && miniHS2.isStarted()); + + try ( + Connection hs2Conn1 = DriverManager + .getConnection(TestHiveMetaStoreAlterColumnPar.miniHS2.getJdbcURL(), "hive", "hive"); + Statement stmt1 = hs2Conn1.createStatement(); + Connection hs2Conn2 = DriverManager + .getConnection(TestHiveMetaStoreAlterColumnPar.miniHS2.getJdbcURL(), "hive", "hive"); + Statement stmt2 = hs2Conn2.createStatement(); + ) { + // Set parameter to be false in connection 1. int to smallint allowed + stmt1.execute("set metaconf:hive.metastore.disallow.incompatible.col.type.changes=false"); + stmt1.execute("alter table t1 change column c1 c1 smallint"); + stmt1.execute("alter table t1 change column c1 c1 int"); + + // parameter value not changed to false in connection 2. int to smallint throws exception + try { + stmt2.execute("alter table t1 change column c1 c1 smallint"); + assertTrue("Exception not thrown", true); + } catch (Exception e1) { + assertTrue("Unexpected exception: " + e1.getMessage(), e1.getMessage().contains( + "Unable to alter table. The following columns have types incompatible with the existing columns in their respective positions")); + } + + // parameter value is still false in 1st connection. The alter still goes through. + stmt1.execute("alter table t1 change column c1 c1 smallint"); + } catch (Exception e2) { + assertTrue("Unexpected Exception: " + e2.getMessage(), true); + } + } +} diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 921cfc00343807179340fbdf40f21e2a46d936ab..bd4a154ec1d7f81823ceb81c53bbef0ff9f71cd8 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -106,7 +106,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, String newTblName = newt.getTableName().toLowerCase(); String newDbName = newt.getDbName().toLowerCase(); - if (!MetaStoreUtils.validateName(newTblName, hiveConf)) { + if (!MetaStoreUtils.validateName(newTblName, handler.getConf())) { throw new InvalidOperationException(newTblName + " is not a valid object name"); } String validate = MetaStoreUtils.validateTblColumns(newt.getSd().getCols()); @@ -155,7 +155,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, // Views derive the column type from the base table definition. So the view definition // can be altered to change the column types. The column type compatibility checks should // be done only for non-views. - if (MetastoreConf.getBoolVar(hiveConf, + if (MetastoreConf.getBoolVar(handler.getConf(), MetastoreConf.ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES) && !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())) { // Throws InvalidOperationException if the new column types are not @@ -276,7 +276,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, } } else { // operations other than table rename - if (MetaStoreUtils.requireCalStats(hiveConf, null, null, newt, environmentContext) && + if (MetaStoreUtils.requireCalStats(handler.getConf(), null, null, newt, environmentContext) && !isPartitionedTable) { Database db = msdb.getDatabase(newDbName); // Update table stats. For partitioned table, we update stats in alterPartition() @@ -422,7 +422,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String try { msdb.openTransaction(); oldPart = msdb.getPartition(dbname, name, new_part.getValues()); - if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) { + if (MetaStoreUtils.requireCalStats(handler.getConf(), oldPart, new_part, tbl, environmentContext)) { // if stats are same, no need to update if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) { MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters()); @@ -555,7 +555,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String new_part.getSd().setLocation(oldPart.getSd().getLocation()); } - if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) { + if (MetaStoreUtils.requireCalStats(handler.getConf(), oldPart, new_part, tbl, environmentContext)) { MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext); } @@ -647,7 +647,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String oldParts.add(oldTmpPart); partValsList.add(tmpPart.getValues()); - if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl, environmentContext)) { + if (MetaStoreUtils.requireCalStats(handler.getConf(), oldTmpPart, tmpPart, tbl, environmentContext)) { // Check if stats are same, no need to update if (MetaStoreUtils.isFastStatsSame(oldTmpPart, tmpPart)) { MetaStoreUtils.updateBasicState(environmentContext, tmpPart.getParameters());