diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 9d58193..bb6f8db 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6903,7 +6903,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) throw new SemanticException("Unknown destination type: " + dest_type); } - input = genConversionSelectOperator(dest, qb, input, table_desc, dpCtx); + input = genConversionSelectOperator(dest, qb, input, table_desc, dest_tab, dpCtx); inputRR = opParseCtx.get(input).getRowResolver(); @@ -7106,12 +7106,42 @@ These props are now enabled elsewhere (see commit diffs). It would be better in } + private ExprNodeDesc convertSelectColumnIfNecessary(String dest, + int i, + ExprNodeDesc column, + TypeInfo rowFieldTypeInfo, + TypeInfo tableFieldTypeInfo, + boolean isLazySimpleSerDe) throws SemanticException { + // LazySimpleSerDe can convert any types to String type using + // JSON-format. + if (!tableFieldTypeInfo.equals(rowFieldTypeInfo) + && !(isLazySimpleSerDe + && tableFieldTypeInfo.getCategory().equals(Category.PRIMITIVE) && tableFieldTypeInfo + .equals(TypeInfoFactory.stringTypeInfo))) { + // need to do some conversions here + if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) { + // cannot convert to complex types + column = null; + } else { + column = ParseUtils.createConversionCast( + column, (PrimitiveTypeInfo)tableFieldTypeInfo); + } + if (column == null) { + String reason = "Cannot convert column " + i + " from " + + rowFieldTypeInfo + " to " + tableFieldTypeInfo + "."; + throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH + .getMsg(qb.getParseInfo().getDestForClause(dest), reason)); + } + } + return column; + } + /** * Generate the conversion SelectOperator that converts the columns into the * types that are expected by the table_desc. */ Operator genConversionSelectOperator(String dest, QB qb, Operator input, - TableDesc table_desc, DynamicPartitionCtx dpCtx) throws SemanticException { + TableDesc table_desc, Table dest_tab, DynamicPartitionCtx dpCtx) throws SemanticException { StructObjectInspector oi = null; try { Deserializer deserializer = table_desc.getDeserializerClass() @@ -7173,42 +7203,37 @@ Operator genConversionSelectOperator(String dest, QB qb, Operator input, ExprNodeDesc column = new ExprNodeColumnDesc(rowFieldTypeInfo, rowFields.get(rowFieldsOffset).getInternalName(), "", false, rowFields.get(rowFieldsOffset).isSkewedCol()); - // LazySimpleSerDe can convert any types to String type using - // JSON-format. - if (!tableFieldTypeInfo.equals(rowFieldTypeInfo) - && !(isLazySimpleSerDe - && tableFieldTypeInfo.getCategory().equals(Category.PRIMITIVE) && tableFieldTypeInfo - .equals(TypeInfoFactory.stringTypeInfo))) { - // need to do some conversions here + ExprNodeDesc convertedColumn = convertSelectColumnIfNecessary( + dest, i, column, rowFieldTypeInfo, tableFieldTypeInfo, isLazySimpleSerDe); + if (convertedColumn != column) { + column = convertedColumn; converted = true; - if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) { - // cannot convert to complex types - column = null; - } else { - column = ParseUtils.createConversionCast( - column, (PrimitiveTypeInfo)tableFieldTypeInfo); - } - if (column == null) { - String reason = "Cannot convert column " + i + " from " - + rowFieldTypeInfo + " to " + tableFieldTypeInfo + "."; - throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH - .getMsg(qb.getParseInfo().getDestForClause(dest), reason)); - } } expressions.add(column); } } - // deal with dynamic partition columns: convert ExprNodeDesc type to String?? + // deal with dynamic partition columns: if (dynPart && dpCtx != null && dpCtx.getNumDPCols() > 0) { // DP columns starts with tableFields.size() - for (int i = tableFields.size() + (updating() ? 1 : 0); i < rowFields.size(); ++i) { + int initialDPColIdx = tableFields.size() + (updating() ? 1 : 0); + for (int i = initialDPColIdx; i < rowFields.size(); ++i) { TypeInfo rowFieldTypeInfo = rowFields.get(i).getType(); ExprNodeDesc column = new ExprNodeColumnDesc( rowFieldTypeInfo, rowFields.get(i).getInternalName(), "", true); + // Dynamic partition column value may need to be converted to the column type + // Does this need to be further converted to String? + String dpColName = dpCtx.getDPColNames().get(i - initialDPColIdx); + FieldSchema dpColFieldSchema = dest_tab.getPartColByName(dpColName); + TypeInfo dpColTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(dpColFieldSchema.getType()); + ExprNodeDesc convertedColumn = convertSelectColumnIfNecessary(dest, i, + column, rowFieldTypeInfo, dpColTypeInfo, isLazySimpleSerDe); + if (convertedColumn != column) { + column = convertedColumn; + converted = true; + } expressions.add(column); } - // converted = true; // [TODO]: should we check & convert type to String and set it to true? } if (converted) { diff --git a/ql/src/test/queries/clientpositive/dynamic_partition_insert2.q b/ql/src/test/queries/clientpositive/dynamic_partition_insert2.q new file mode 100644 index 0000000..56ece7a --- /dev/null +++ b/ql/src/test/queries/clientpositive/dynamic_partition_insert2.q @@ -0,0 +1,22 @@ +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.fetch.task.conversion=none; + +drop table if exists iow1; +create table iow1(key int) partitioned by (key2 int); + +select key, key + 1 as k1, key + 1 as k2 from src where key >= 0 order by k1 desc limit 1; + +explain +insert overwrite table iow1 partition (key2) +select key + 1 as k1, key + 1 as k2 from src where key >= 0 order by k1 desc limit 1; + +insert overwrite table iow1 partition (key2) +select key + 1 as k1, key + 1 as k2 from src where key >= 0 order by k1 desc limit 1; + +show partitions iow1; + +select key, key2 from iow1; + +drop table iow1; diff --git a/ql/src/test/results/clientpositive/dynamic_partition_insert2.q.out b/ql/src/test/results/clientpositive/dynamic_partition_insert2.q.out new file mode 100644 index 0000000..438cbd1 --- /dev/null +++ b/ql/src/test/results/clientpositive/dynamic_partition_insert2.q.out @@ -0,0 +1,126 @@ +PREHOOK: query: drop table if exists iow1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists iow1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table iow1(key int) partitioned by (key2 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@iow1 +POSTHOOK: query: create table iow1(key int) partitioned by (key2 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@iow1 +PREHOOK: query: select key, key + 1 as k1, key + 1 as k2 from src where key >= 0 order by k1 desc limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select key, key + 1 as k1, key + 1 as k2 from src where key >= 0 order by k1 desc limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +498 499.0 499.0 +PREHOOK: query: explain +insert overwrite table iow1 partition (key2) +select key + 1 as k1, key + 1 as k2 from src where key >= 0 order by k1 desc limit 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table iow1 partition (key2) +select key + 1 as k1, key + 1 as k2 from src where key >= 0 order by k1 desc limit 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) >= 0.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (UDFToDouble(key) + 1.0) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: - + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: double), KEY.reducesinkkey0 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(_col0) (type: int), UDFToInteger(_col1) (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.iow1 + + Stage: Stage-0 + Move Operator + tables: + partition: + key2 + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.iow1 + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: insert overwrite table iow1 partition (key2) +select key + 1 as k1, key + 1 as k2 from src where key >= 0 order by k1 desc limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@iow1 +POSTHOOK: query: insert overwrite table iow1 partition (key2) +select key + 1 as k1, key + 1 as k2 from src where key >= 0 order by k1 desc limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@iow1@key2=499 +POSTHOOK: Lineage: iow1 PARTITION(key2=499).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: show partitions iow1 +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@iow1 +POSTHOOK: query: show partitions iow1 +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@iow1 +key2=499 +PREHOOK: query: select key, key2 from iow1 +PREHOOK: type: QUERY +PREHOOK: Input: default@iow1 +PREHOOK: Input: default@iow1@key2=499 +#### A masked pattern was here #### +POSTHOOK: query: select key, key2 from iow1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@iow1 +POSTHOOK: Input: default@iow1@key2=499 +#### A masked pattern was here #### +499 499 +PREHOOK: query: drop table iow1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@iow1 +PREHOOK: Output: default@iow1 +POSTHOOK: query: drop table iow1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@iow1 +POSTHOOK: Output: default@iow1