Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1162190) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -2722,7 +2722,6 @@ String afterCol = alterTbl.getAfterCol(); FieldSchema column = null; - boolean found = false; int position = -1; if (first) { position = 0; @@ -2732,11 +2731,7 @@ while (iterOldCols.hasNext()) { FieldSchema col = iterOldCols.next(); String oldColName = col.getName(); - if (oldColName.equalsIgnoreCase(newName) - && !oldColName.equalsIgnoreCase(oldName)) { - console.printError("Column '" + newName + "' exists"); - return 1; - } else if (oldColName.equalsIgnoreCase(oldName)) { + if (oldColName.equalsIgnoreCase(oldName)) { col.setName(newName); if (type != null && !type.trim().equals("")) { col.setType(type); @@ -2744,7 +2739,6 @@ if (comment != null) { col.setComment(comment); } - found = true; if (first || (afterCol != null && !afterCol.trim().equals(""))) { column = col; continue; @@ -2760,17 +2754,6 @@ newCols.add(col); } - // did not find the column - if (!found) { - console.printError("Column '" + oldName + "' does not exist"); - return 1; - } - // after column is not null, but we did not find it. - if ((afterCol != null && !afterCol.trim().equals("")) && position < 0) { - console.printError("Column '" + afterCol + "' does not exist"); - return 1; - } - if (position >= 0) { newCols.add(position, column); } @@ -2783,13 +2766,9 @@ console .printInfo("Replacing columns for columnsetSerDe and changing to LazySimpleSerDe"); tbl.setSerializationLib(LazySimpleSerDe.class.getName()); - } else if (!tbl.getSerializationLib().equals( - MetadataTypedColumnsetSerDe.class.getName()) - && !tbl.getSerializationLib().equals(LazySimpleSerDe.class.getName()) - && !tbl.getSerializationLib().equals(ColumnarSerDe.class.getName()) - && !tbl.getSerializationLib().equals(DynamicSerDe.class.getName())) { + } else if (!SerDeUtils.nativeSerDeNames.contains(tbl.getSerializationLib())) { console.printError("Replace columns is not supported for this table. " - + "SerDe may be incompatible."); + + "SerDe: " + tbl.getSerializationLib() + " may be incompatible."); return 1; } tbl.getTTable().getSd().setCols(alterTbl.getNewCols()); Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (revision 1162190) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (working copy) @@ -260,6 +260,10 @@ } return deserializer; } + + public void clearDeserializer() { + deserializer = null; + } public HiveStorageHandler getStorageHandler() { if (storageHandler != null) { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1162190) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -1687,15 +1687,20 @@ unescapeIdentifier(ast.getChild(1).getText()), unescapeIdentifier(ast .getChild(2).getText()), newType, newComment, first, flagCol); + Table tab = null; try { - Table tab = db.getTable(db.getCurrentDatabase(), tblName, false); - if (tab != null) { - inputs.add(new ReadEntity(tab)); - outputs.add(new WriteEntity(tab)); - } + tab = db.getTable(db.getCurrentDatabase(), tblName, false); } catch (HiveException e) { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tblName)); } + + this.validateRenameColumn(alterTblDesc, tab); + + if (tab != null) { + inputs.add(new ReadEntity(tab)); + outputs.add(new WriteEntity(tab)); + } + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc), conf)); @@ -1708,20 +1713,83 @@ AlterTableDesc alterTblDesc = new AlterTableDesc(tblName, newCols, alterType); + Table tab = null; try { - Table tab = db.getTable(db.getCurrentDatabase(), tblName, false); - if (tab != null) { - inputs.add(new ReadEntity(tab)); - outputs.add(new WriteEntity(tab)); - } + tab = db.getTable(db.getCurrentDatabase(), tblName, false); } catch (HiveException e) { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tblName)); } + if (alterTblDesc.getOp() == AlterTableDesc.AlterTableTypes.RENAMECOLUMN) { + validateRenameColumn(alterTblDesc, tab); + } else if (alterTblDesc.getOp() == AlterTableDesc.AlterTableTypes.REPLACECOLS) { + validateReplaceColumns(alterTblDesc, tab); + } + + if (tab != null) { + inputs.add(new ReadEntity(tab)); + outputs.add(new WriteEntity(tab)); + } + + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc), conf)); } + private void validateReplaceColumns(AlterTableDesc alterTblDesc, Table tab) + throws SemanticException { + tab.getTTable().getSd().setCols(alterTblDesc.getNewCols()); + tab.clearDeserializer(); + try { + tab.checkValidity(); + } catch (HiveException e) { + throw new SemanticException(ErrorMsg.INVALID_COLUMN + .getMsg(e.getMessage())); + } + } + + private void validateRenameColumn(AlterTableDesc alterTblDesc, Table tab) + throws SemanticException { + List oldCols = tab.getCols(); + Iterator iterOldCols = oldCols.iterator(); + String oldName = alterTblDesc.getOldColName(); + String newName = alterTblDesc.getNewColName(); + String afterCol = alterTblDesc.getAfterCol(); + + boolean found = false; + int position = -1; + + int i = 0; + while (iterOldCols.hasNext()) { + FieldSchema col = iterOldCols.next(); + String oldColName = col.getName(); + if (oldColName.equalsIgnoreCase(newName) + && !oldColName.equalsIgnoreCase(oldName)) { + throw new SemanticException(ErrorMsg.DUPLICATE_COLUMN_NAMES + .getMsg(newName)); + } + + if(oldName.equalsIgnoreCase(oldColName)) { + found = true; + } + + if (afterCol != null && !afterCol.trim().equals("") + && oldColName.equalsIgnoreCase(afterCol)) { + position = i; + } + + i++; + } + // did not find the column + if (!found) { + throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(oldName)); + } + // after column is not null, but we did not find it. + if ((afterCol != null && !afterCol.trim().equals("")) && position < 0) { + throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(afterCol)); + } + } + private void analyzeAlterTableDropParts(ASTNode ast, boolean expectView) throws SemanticException { Index: ql/src/test/queries/clientnegative/replace_columns.q =================================================================== --- ql/src/test/queries/clientnegative/replace_columns.q (revision 0) +++ ql/src/test/queries/clientnegative/replace_columns.q (revision 0) @@ -0,0 +1,3 @@ +create table replace_column_part (key int, value string) partitioned by (ds string) stored as textfile; +show table extended like replace_column_part; +alter table replace_column_part replace columns (key int, value string, ds string); Index: ql/src/test/queries/clientnegative/replace_columns_2.q =================================================================== --- ql/src/test/queries/clientnegative/replace_columns_2.q (revision 0) +++ ql/src/test/queries/clientnegative/replace_columns_2.q (revision 0) @@ -0,0 +1,3 @@ +create table replace_column_part (key int, value string) partitioned by (ds string) stored as rcfile; +show table extended like replace_column_part; +alter table replace_column_part replace columns (key int, value string, ds string); \ No newline at end of file Index: ql/src/test/queries/clientnegative/replace_columns_3.q =================================================================== --- ql/src/test/queries/clientnegative/replace_columns_3.q (revision 0) +++ ql/src/test/queries/clientnegative/replace_columns_3.q (revision 0) @@ -0,0 +1,4 @@ +create table replace_column_part (key int, value string) partitioned by (ds string) stored as rcfile; +alter table replace_column_part set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'; +show table extended like replace_column_part; +alter table replace_column_part replace columns (key int, value string, ds string); \ No newline at end of file Index: ql/src/test/results/clientnegative/altern1.q.out =================================================================== --- ql/src/test/results/clientnegative/altern1.q.out (revision 1162190) +++ ql/src/test/results/clientnegative/altern1.q.out (working copy) @@ -3,9 +3,4 @@ POSTHOOK: query: create table altern1(a int, b int) partitioned by (ds string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@altern1 -PREHOOK: query: alter table altern1 replace columns(a int, b int, ds string) -PREHOOK: type: ALTERTABLE_REPLACECOLS -PREHOOK: Input: default@altern1 -PREHOOK: Output: default@altern1 -Invalid table columns : Partition column name ds conflicts with table columns. -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask +FAILED: Error in semantic analysis: Invalid column reference Partition column name ds conflicts with table columns. Index: ql/src/test/results/clientnegative/column_rename1.q.out =================================================================== --- ql/src/test/results/clientnegative/column_rename1.q.out (revision 1162190) +++ ql/src/test/results/clientnegative/column_rename1.q.out (working copy) @@ -19,9 +19,4 @@ POSTHOOK: Output: default@tstsrc POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: alter table tstsrc change src_not_exist key_value string -PREHOOK: type: ALTERTABLE_RENAMECOL -PREHOOK: Input: default@tstsrc -PREHOOK: Output: default@tstsrc -Column 'src_not_exist' does not exist -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask +FAILED: Error in semantic analysis: Invalid column reference src_not_exist Index: ql/src/test/results/clientnegative/column_rename2.q.out =================================================================== --- ql/src/test/results/clientnegative/column_rename2.q.out (revision 1162190) +++ ql/src/test/results/clientnegative/column_rename2.q.out (working copy) @@ -19,9 +19,4 @@ POSTHOOK: Output: default@tstsrc POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: alter table tstsrc change key value string -PREHOOK: type: ALTERTABLE_RENAMECOL -PREHOOK: Input: default@tstsrc -PREHOOK: Output: default@tstsrc -Column 'value' exists -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask +FAILED: Error in semantic analysis: Duplicate column name: value Index: ql/src/test/results/clientnegative/column_rename4.q.out =================================================================== --- ql/src/test/results/clientnegative/column_rename4.q.out (revision 1162190) +++ ql/src/test/results/clientnegative/column_rename4.q.out (working copy) @@ -19,9 +19,4 @@ POSTHOOK: Output: default@tstsrc POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: alter table tstsrc change key key2 string after key_value -PREHOOK: type: ALTERTABLE_RENAMECOL -PREHOOK: Input: default@tstsrc -PREHOOK: Output: default@tstsrc -Column 'key_value' does not exist -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask +FAILED: Error in semantic analysis: Invalid column reference key_value Index: ql/src/test/results/clientnegative/replace_columns.q.out =================================================================== --- ql/src/test/results/clientnegative/replace_columns.q.out (revision 0) +++ ql/src/test/results/clientnegative/replace_columns.q.out (revision 0) @@ -0,0 +1,19 @@ +PREHOOK: query: create table replace_column_part (key int, value string) partitioned by (ds string) stored as textfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table replace_column_part (key int, value string) partitioned by (ds string) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@replace_column_part +PREHOOK: query: show table extended like replace_column_part +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like replace_column_part +POSTHOOK: type: SHOW_TABLESTATUS +tableName:replace_column_part +owner:heyongqiang +location:pfile:/Users/heyongqiang/Documents/workspace/Hive-3/build/ql/test/data/warehouse/replace_column_part +inputformat:org.apache.hadoop.mapred.TextInputFormat +outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} + +FAILED: Error in semantic analysis: Invalid column reference Partition column name ds conflicts with table columns. Index: ql/src/test/results/clientnegative/replace_columns_2.q.out =================================================================== --- ql/src/test/results/clientnegative/replace_columns_2.q.out (revision 0) +++ ql/src/test/results/clientnegative/replace_columns_2.q.out (revision 0) @@ -0,0 +1,19 @@ +PREHOOK: query: create table replace_column_part (key int, value string) partitioned by (ds string) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table replace_column_part (key int, value string) partitioned by (ds string) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@replace_column_part +PREHOOK: query: show table extended like replace_column_part +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like replace_column_part +POSTHOOK: type: SHOW_TABLESTATUS +tableName:replace_column_part +owner:heyongqiang +location:pfile:/Users/heyongqiang/Documents/workspace/Hive-3/build/ql/test/data/warehouse/replace_column_part +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} + +FAILED: Error in semantic analysis: Invalid column reference Partition column name ds conflicts with table columns. Index: ql/src/test/results/clientnegative/replace_columns_3.q.out =================================================================== --- ql/src/test/results/clientnegative/replace_columns_3.q.out (revision 0) +++ ql/src/test/results/clientnegative/replace_columns_3.q.out (revision 0) @@ -0,0 +1,27 @@ +PREHOOK: query: create table replace_column_part (key int, value string) partitioned by (ds string) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table replace_column_part (key int, value string) partitioned by (ds string) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@replace_column_part +PREHOOK: query: alter table replace_column_part set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@replace_column_part +PREHOOK: Output: default@replace_column_part +POSTHOOK: query: alter table replace_column_part set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@replace_column_part +POSTHOOK: Output: default@replace_column_part +PREHOOK: query: show table extended like replace_column_part +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like replace_column_part +POSTHOOK: type: SHOW_TABLESTATUS +tableName:replace_column_part +owner:heyongqiang +location:pfile:/Users/heyongqiang/Documents/workspace/Hive-3/build/ql/test/data/warehouse/replace_column_part +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} + +FAILED: Error in semantic analysis: Invalid column reference Partition column name ds conflicts with table columns. Index: ql/src/test/results/clientnegative/replace_columns_4.q.out =================================================================== --- ql/src/test/results/clientnegative/replace_columns_4.q.out (revision 0) +++ ql/src/test/results/clientnegative/replace_columns_4.q.out (revision 0) @@ -0,0 +1,17 @@ +PREHOOK: query: create table replace_column_part (key int, value string) partitioned by (ds string) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table replace_column_part (key int, value string) partitioned by (ds string) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@replace_column_part +PREHOOK: query: alter table replace_column_part set serde 'org.apache.hadoop.hive.serde2.DummySerde' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@replace_column_part +PREHOOK: Output: default@replace_column_part +POSTHOOK: query: alter table replace_column_part set serde 'org.apache.hadoop.hive.serde2.DummySerde' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@replace_column_part +POSTHOOK: Output: default@replace_column_part +PREHOOK: query: show table extended like replace_column_part +PREHOOK: type: SHOW_TABLESTATUS +Failed with exception MetaException(message:org.apache.hadoop.hive.serde2.SerDeException SerDe org.apache.hadoop.hive.serde2.DummySerde does not exist) +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (revision 1162190) +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (working copy) @@ -83,7 +83,7 @@ } } - private static List nativeSerDeNames = new ArrayList(); + public static List nativeSerDeNames = new ArrayList(); static { nativeSerDeNames .add(org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe.class @@ -96,6 +96,10 @@ nativeSerDeNames .add(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); nativeSerDeNames.add(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName()); + nativeSerDeNames + .add(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName()); + nativeSerDeNames + .add(org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName()); } public static boolean shouldGetColsFromSerDe(String serde) {