diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 4d8e10c..40668b0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -3745,8 +3745,7 @@ private int alterTable(Hive db, AlterTableDesc alterTbl) throws HiveException { tbl.getTTable().getSd().getSerdeInfo().getParameters().putAll( alterTbl.getProps()); } - if (!conf.getStringCollection(ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname) - .contains(serdeName)) { + if (!Table.hasMetastoreBasedSchema(conf, serdeName)) { tbl.setFields(Hive.getFieldsFromDeserializer(tbl.getTableName(), tbl. getDeserializer())); } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 250756c..fcf082b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -402,6 +402,7 @@ public void alterTable(String tblName, Table newTbl) if (newTbl.getParameters() != null) { newTbl.getParameters().remove(hive_metastoreConstants.DDL_TIME); } + newTbl.checkValidity(); getMSC().alter_table(names[0], names[1], newTbl.getTTable()); } catch (MetaException e) { throw new HiveException("Unable to alter table.", e); @@ -469,6 +470,7 @@ public void alterPartition(String dbName, String tblName, Partition newPart) if (newPart.getParameters() != null) { newPart.getParameters().remove(hive_metastoreConstants.DDL_TIME); } + newPart.checkValidity(); getMSC().alter_partition(dbName, tblName, newPart.getTPartition()); } catch (MetaException e) { @@ -1014,10 +1016,7 @@ public Table getTable(final String dbName, final String tableName, } } - Table table = new Table(tTable); - - table.checkValidity(); - return table; + return new Table(tTable); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index 3a1e7fd..44f6198 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.JavaUtils; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.ProtectMode; import org.apache.hadoop.hive.metastore.Warehouse; @@ -46,7 +45,6 @@ import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; import org.apache.hadoop.hive.serde2.Deserializer; -import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.mapred.InputFormat; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; @@ -504,8 +502,7 @@ public void setTPartition( public List getCols() { try { - if (Hive.get().getConf().getStringCollection(ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname) - .contains(tPartition.getSd().getSerdeInfo().getSerializationLib())) { + if (Table.hasMetastoreBasedSchema(Hive.get().getConf(), tPartition.getSd())) { return tPartition.getSd().getCols(); } return Hive.getFieldsFromDeserializer(table.getTableName(), getDeserializer()); @@ -644,4 +641,10 @@ public void setSkewedValueLocationMap(List valList, String dirName) public Map, String> getSkewedColValueLocationMaps() { return tPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps(); } + + public void checkValidity() throws HiveException { + if (!tPartition.getSd().equals(table.getSd())) { + Table.validateColumns(getCols(), table.getPartCols()); + } + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 3df2690..3ebb9b1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -29,12 +29,14 @@ import java.util.Map; import java.util.Properties; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.JavaUtils; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.ProtectMode; @@ -58,7 +60,6 @@ import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.Writable; @@ -210,37 +211,11 @@ public void checkValidity() throws HiveException { assert(getViewExpandedText() == null); } - Iterator iterCols = getCols().iterator(); - List colNames = new ArrayList(); - while (iterCols.hasNext()) { - String colName = iterCols.next().getName(); - if (!MetaStoreUtils.validateColumnName(colName)) { - throw new HiveException("Invalid column name '" + colName - + "' in the table definition"); - } - Iterator iter = colNames.iterator(); - while (iter.hasNext()) { - String oldColName = iter.next(); - if (colName.equalsIgnoreCase(oldColName)) { - throw new HiveException("Duplicate column name " + colName - + " in the table definition."); - } - } - colNames.add(colName.toLowerCase()); - } + validateColumns(getCols(), getPartCols()); + } - if (getPartCols() != null) { - // there is no overlap between columns and partitioning columns - Iterator partColsIter = getPartCols().iterator(); - while (partColsIter.hasNext()) { - String partCol = partColsIter.next().getName(); - if (colNames.contains(partCol.toLowerCase())) { - throw new HiveException("Partition column name " + partCol - + " conflicts with table columns."); - } - } - } - return; + public StorageDescriptor getSd() { + return tTable.getSd(); } public void setInputFormatClass(Class inputFormatClass) { @@ -623,15 +598,15 @@ private boolean isField(String col) { public List getCols() { + String serializationLib = getSerializationLib(); try { - if (null == getSerializationLib() || Hive.get().getConf().getStringCollection( - ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname).contains(getSerializationLib())) { + if (hasMetastoreBasedSchema(Hive.get().getConf(), serializationLib)) { return tTable.getSd().getCols(); } else { return Hive.getFieldsFromDeserializer(getTableName(), getDeserializer()); } } catch (HiveException e) { - LOG.error("Unable to get field from serde: " + getSerializationLib(), e); + LOG.error("Unable to get field from serde: " + serializationLib, e); } return new ArrayList(); } @@ -996,4 +971,44 @@ public String getCompleteName() { throw new RuntimeException("Cannot get path ", e); } } + + public static boolean hasMetastoreBasedSchema(HiveConf conf, StorageDescriptor serde) { + return hasMetastoreBasedSchema(conf, serde.getSerdeInfo().getSerializationLib()); + } + + public static boolean hasMetastoreBasedSchema(HiveConf conf, String serdeLib) { + return StringUtils.isEmpty(serdeLib) || + conf.getStringCollection(ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname).contains(serdeLib); + } + + public static void validateColumns(List columns, List partCols) + throws HiveException { + List colNames = new ArrayList(); + for (FieldSchema partCol: columns) { + String colName = partCol.getName(); + if (!MetaStoreUtils.validateColumnName(colName)) { + throw new HiveException("Invalid column name '" + colName + + "' in the table definition"); + } + Iterator iter = colNames.iterator(); + while (iter.hasNext()) { + String oldColName = iter.next(); + if (colName.equalsIgnoreCase(oldColName)) { + throw new HiveException("Duplicate column name " + colName + + " in the table definition."); + } + } + colNames.add(colName.toLowerCase()); + } + if (partCols != null) { + // there is no overlap between columns and partitioning columns + for (FieldSchema partCol: partCols) { + if (colNames.contains(partCol.getName().toLowerCase())) { + throw new HiveException("Partition column name " + partCol + + " conflicts with table columns."); + } + } + } + } + }; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java index 2537b75..c059632 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java @@ -24,22 +24,20 @@ import java.util.List; import java.util.Map; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -410,8 +408,7 @@ public void validate(HiveConf conf) if ((this.getCols() == null) || (this.getCols().size() == 0)) { // for now make sure that serde exists - if (StringUtils.isEmpty(this.getSerName()) - || conf.getStringCollection(ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname).contains(this.getSerName())) { + if (Table.hasMetastoreBasedSchema(conf, getSerName())) { throw new SemanticException(ErrorMsg.INVALID_TBL_DDL_SERDE.getMsg()); } return;