diff --git a/metastore/scripts/upgrade/mysql/015-HIVE-5700.mysql.sql b/metastore/scripts/upgrade/mysql/015-HIVE-5700.mysql.sql deleted file mode 100644 index dd199c1..0000000 --- a/metastore/scripts/upgrade/mysql/015-HIVE-5700.mysql.sql +++ /dev/null @@ -1,11 +0,0 @@ -SELECT '< HIVE-5700 enforce single date format for partition column storage >' AS ' '; - --- Normalize the date partition column values as best we can. No schema changes. - - -UPDATE PARTITION_KEY_VALS - INNER JOIN PARTITIONS ON PARTITION_KEY_VALS.PART_ID = PARTITIONS.PART_ID - INNER JOIN PARTITION_KEYS ON PARTITION_KEYS.TBL_ID = PARTITIONS.TBL_ID - AND PARTITION_KEYS.INTEGER_IDX = PARTITION_KEY_VALS.INTEGER_IDX - AND PARTITION_KEYS.PKEY_TYPE = 'date' -SET PART_KEY_VAL = IFNULL(DATE_FORMAT(cast(PART_KEY_VAL as date),'%Y-%m-%d'), PART_KEY_VAL); diff --git a/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql index e324bda..e511b3f 100644 --- a/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/upgrade-0.12.0-to-0.13.0.mysql.sql @@ -1,6 +1,5 @@ SELECT 'Upgrading MetaStore schema from 0.12.0 to 0.13.0' AS ' '; -SOURCE 015-HIVE-5700.mysql.sql; SOURCE 016-HIVE-6386.mysql.sql; SOURCE 017-HIVE-6458.mysql.sql; SOURCE 018-HIVE-6757.mysql.sql; diff --git a/metastore/scripts/upgrade/oracle/015-HIVE-5700.oracle.sql b/metastore/scripts/upgrade/oracle/015-HIVE-5700.oracle.sql deleted file mode 100644 index 0414fc6..0000000 --- a/metastore/scripts/upgrade/oracle/015-HIVE-5700.oracle.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Normalize the date partition column values as best we can. No schema changes. - -CREATE FUNCTION hive13_to_date(date_str IN VARCHAR2) RETURN DATE IS dt DATE; BEGIN dt := TO_DATE(date_str, 'YYYY-MM-DD'); RETURN dt; EXCEPTION WHEN others THEN RETURN null; END;/ - -MERGE INTO PARTITION_KEY_VALS -USING ( - SELECT SRC.PART_ID as IPART_ID, SRC.INTEGER_IDX as IINTEGER_IDX, - NVL(TO_CHAR(hive13_to_date(PART_KEY_VAL),'YYYY-MM-DD'), PART_KEY_VAL) as NORM - FROM PARTITION_KEY_VALS SRC - INNER JOIN PARTITIONS ON SRC.PART_ID = PARTITIONS.PART_ID - INNER JOIN PARTITION_KEYS ON PARTITION_KEYS.TBL_ID = PARTITIONS.TBL_ID - AND PARTITION_KEYS.INTEGER_IDX = SRC.INTEGER_IDX AND PARTITION_KEYS.PKEY_TYPE = 'date' -) ON (IPART_ID = PARTITION_KEY_VALS.PART_ID AND IINTEGER_IDX = PARTITION_KEY_VALS.INTEGER_IDX) -WHEN MATCHED THEN UPDATE SET PART_KEY_VAL = NORM; - -DROP FUNCTION hive13_to_date; - diff --git a/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql index 5739768..18aeddf 100644 --- a/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql @@ -1,6 +1,5 @@ SELECT 'Upgrading MetaStore schema from 0.12.0 to 0.13.0' AS Status from dual; -@015-HIVE-5700.oracle.sql; @016-HIVE-6386.oracle.sql; @017-HIVE-6458.oracle.sql; @018-HIVE-6757.oracle.sql; diff --git a/metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql b/metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql deleted file mode 100644 index aedb2b5..0000000 --- a/metastore/scripts/upgrade/postgres/015-HIVE-5700.postgres.sql +++ /dev/null @@ -1,15 +0,0 @@ -SELECT '< HIVE-5700 enforce single date format for partition column storage >'; - --- Normalize the date partition column values as best we can. No schema changes. -CREATE FUNCTION hive13_to_date(date_str text) RETURNS DATE AS $$ DECLARE dt DATE; BEGIN dt := date_str::DATE; RETURN dt; EXCEPTION WHEN others THEN RETURN null; END; $$ LANGUAGE plpgsql; - -UPDATE "PARTITION_KEY_VALS" -SET "PART_KEY_VAL" = COALESCE(TO_CHAR(hive13_to_date(src."PART_KEY_VAL"),'YYYY-MM-DD'), src."PART_KEY_VAL") -FROM "PARTITION_KEY_VALS" src - INNER JOIN "PARTITIONS" ON src."PART_ID" = "PARTITIONS"."PART_ID" - INNER JOIN "PARTITION_KEYS" ON "PARTITION_KEYS"."TBL_ID" = "PARTITIONS"."TBL_ID" - AND "PARTITION_KEYS"."INTEGER_IDX" = src."INTEGER_IDX" - AND "PARTITION_KEYS"."PKEY_TYPE" = 'date'; - -DROP FUNCTION hive13_to_date(date_str text); - diff --git a/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql index 72c9b40..b0394e0 100644 --- a/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/upgrade-0.12.0-to-0.13.0.postgres.sql @@ -1,6 +1,5 @@ SELECT 'Upgrading MetaStore schema from 0.12.0 to 0.13.0'; -\i 015-HIVE-5700.postgres.sql; \i 016-HIVE-6386.postgres.sql; \i 017-HIVE-6458.postgres.sql; \i 018-HIVE-6757.postgres.sql; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 6b4e06e..a0459c4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -75,8 +75,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import com.google.common.annotations.VisibleForTesting; - /** * BaseSemanticAnalyzer. * @@ -1247,36 +1245,7 @@ public static void validatePartSpec(Table tbl, Map partSpec, inputOI.getTypeName(), outputOI.getTypeName()); } - normalizeColSpec(partSpec, astKeyName, colType, colSpec, convertedValue); - } - } - - @VisibleForTesting - static void normalizeColSpec(Map partSpec, String colName, - String colType, String originalColSpec, Object colValue) throws SemanticException { - if (colValue == null) return; // nothing to do with nulls - String normalizedColSpec = originalColSpec; - if (colType.equals(serdeConstants.DATE_TYPE_NAME)) { - normalizedColSpec = normalizeDateCol(colValue, originalColSpec); - } - if (!normalizedColSpec.equals(originalColSpec)) { - STATIC_LOG.warn("Normalizing partition spec - " + colName + " from " - + originalColSpec + " to " + normalizedColSpec); - partSpec.put(colName, normalizedColSpec); - } - } - - private static String normalizeDateCol( - Object colValue, String originalColSpec) throws SemanticException { - Date value; - if (colValue instanceof DateWritable) { - value = ((DateWritable) colValue).get(); - } else if (colValue instanceof Date) { - value = (Date) colValue; - } else { - throw new SemanticException("Unexpected date type " + colValue.getClass()); } - return HiveMetaStore.PARTITION_DATE_FORMAT.get().format(value); } protected WriteEntity toWriteEntity(String location) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 4364f28..c05b2aa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1340,11 +1340,13 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plan case HiveParser.TOK_INSERT: ASTNode destination = (ASTNode) ast.getChild(0); Tree tab = destination.getChild(0); + // Proceed if AST contains partition & If Not Exists if (destination.getChildCount() == 2 && tab.getChildCount() == 2 && destination.getChild(1).getType() == HiveParser.TOK_IFNOTEXISTS) { String tableName = tab.getChild(0).getChild(0).getText(); + Tree partitions = tab.getChild(1); int childCount = partitions.getChildCount(); HashMap partition = new HashMap(); @@ -1358,30 +1360,25 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plan partition.put(partitionName, partitionVal); } // if it is a dynamic partition throw the exception - if (childCount != partition.size()) { + if (childCount == partition.size()) { + try { + Table table = db.getTable(tableName); + Partition parMetaData = db.getPartition(table, partition, false); + // Check partition exists if it exists skip the overwrite + if (parMetaData != null) { + phase1Result = false; + skipRecursion = true; + LOG.info("Partition already exists so insert into overwrite " + + "skipped for partition : " + parMetaData.toString()); + break; + } + } catch (HiveException e) { + LOG.info("Error while getting metadata : ", e); + } + } else { throw new SemanticException(ErrorMsg.INSERT_INTO_DYNAMICPARTITION_IFNOTEXISTS .getMsg(partition.toString())); } - Table table = null; - try { - table = db.getTable(tableName); - } catch (HiveException ex) { - throw new SemanticException(ex); - } - try { - Partition parMetaData = db.getPartition(table, partition, false); - // Check partition exists if it exists skip the overwrite - if (parMetaData != null) { - phase1Result = false; - skipRecursion = true; - LOG.info("Partition already exists so insert into overwrite " + - "skipped for partition : " + parMetaData.toString()); - break; - } - } catch (HiveException e) { - LOG.info("Error while getting metadata : ", e); - } - validatePartSpec(table, partition, (ASTNode)tab, conf, false); } skipRecursion = false; break; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java deleted file mode 100644 index be1f7ff..0000000 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.parse; - -import static org.junit.Assert.*; - -import java.sql.Date; -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.junit.Test; - -public class TestSemanticAnalyzer { - - @Test - public void testNormalizeColSpec() throws Exception { - // Hive normalizes partition spec for dates to yyyy-mm-dd format. Some versions of Java will - // accept other formats for Date.valueOf, e.g. yyyy-m-d, and who knows what else in the future; - // some will not accept other formats, so we cannot test normalization with them - type check - // will fail before it can ever happen. Thus, test in isolation. - checkNormalization("date", "2010-01-01", "2010-01-01", Date.valueOf("2010-01-01")); - checkNormalization("date", "2010-1-01", "2010-01-01", Date.valueOf("2010-01-01")); - checkNormalization("date", "2010-1-1", "2010-01-01", Date.valueOf("2010-01-01")); - checkNormalization("string", "2010-1-1", "2010-1-1", "2010-1-1"); - - try { - checkNormalization("date", "foo", "", "foo"); // Bad format. - fail("should throw"); - } catch (SemanticException ex) { - } - - try { - checkNormalization("date", "2010-01-01", "2010-01-01", "2010-01-01"); // Bad value type. - fail("should throw"); - } catch (SemanticException ex) { - } - } - - - public void checkNormalization(String colType, String originalColSpec, - String result, Object colValue) throws SemanticException { - final String colName = "col"; - Map partSpec = new HashMap(); - partSpec.put(colName, originalColSpec); - BaseSemanticAnalyzer.normalizeColSpec(partSpec, colName, colType, originalColSpec, colValue); - assertEquals(result, partSpec.get(colName)); - if (colValue instanceof Date) { - DateWritable dw = new DateWritable((Date)colValue); - BaseSemanticAnalyzer.normalizeColSpec(partSpec, colName, colType, originalColSpec, dw); - assertEquals(result, partSpec.get(colName)); - } - } -} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java index 5f0659b..7138d51 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java @@ -49,7 +49,7 @@ public class TestUpdateDeleteSemanticAnalyzer { - static final private Log LOG = LogFactory.getLog(TestSemanticAnalyzer.class.getName()); + static final private Log LOG = LogFactory.getLog(TestUpdateDeleteSemanticAnalyzer.class.getName()); private HiveConf conf; private Hive db;