diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 9889cfe..c2f7b7e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -84,7 +84,7 @@ INVALID_PATH(10027, "Invalid path"), ILLEGAL_PATH(10028, "Path is not legal"), INVALID_NUMERICAL_CONSTANT(10029, "Invalid numerical constant"), - INVALID_ARRAYINDEX_CONSTANT(10030, "Non-constant expressions for array indexes not supported"), + INVALID_ARRAYINDEX_TYPE(10030, "Not proper type for index of ARRAY"), INVALID_MAPINDEX_CONSTANT(10031, "Non-constant expression for map indexes not supported"), INVALID_MAPINDEX_TYPE(10032, "MAP key type does not match index expression type"), NON_COLLECTION_TYPE(10033, "[] not valid on non-collection types"), diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index e7da289..28bdb85 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -63,6 +63,7 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; @@ -751,12 +752,9 @@ static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, if (myt.getCategory() == Category.LIST) { // Only allow integer index for now - if (!(children.get(1) instanceof ExprNodeConstantDesc) - || !(((ExprNodeConstantDesc) children.get(1)).getTypeInfo() - .equals(TypeInfoFactory.intTypeInfo))) { + if (!PrimitiveObjectInspectorUtils.isConvertibleToInteger(children.get(1).getTypeInfo())) { throw new SemanticException(SemanticAnalyzer.generateErrorMessage( - expr, - ErrorMsg.INVALID_ARRAYINDEX_CONSTANT.getMsg())); + expr, ErrorMsg.INVALID_ARRAYINDEX_TYPE.getMsg())); } // Calculate TypeInfo @@ -764,14 +762,7 @@ static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry .getGenericUDFForIndex(), children); } else if (myt.getCategory() == Category.MAP) { - // Only allow constant map key for now - if (!(children.get(1) instanceof ExprNodeConstantDesc)) { - throw new SemanticException(SemanticAnalyzer.generateErrorMessage( - expr, - ErrorMsg.INVALID_MAPINDEX_CONSTANT.getMsg())); - } - if (!(((ExprNodeConstantDesc) children.get(1)).getTypeInfo() - .equals(((MapTypeInfo) myt).getMapKeyTypeInfo()))) { + if (children.get(1).getTypeInfo() != ((MapTypeInfo) myt).getMapKeyTypeInfo()) { throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE .getMsg(expr)); } diff --git ql/src/test/queries/clientpositive/array_map_access_nonconstant.q ql/src/test/queries/clientpositive/array_map_access_nonconstant.q new file mode 100644 index 0000000..d7b64d2 --- /dev/null +++ ql/src/test/queries/clientpositive/array_map_access_nonconstant.q @@ -0,0 +1,15 @@ +set hive.fetch.task.conversion=more; + +create table array_table (array array, index int ); +insert into table array_table select array('first', 'second', 'third'), key%3 from src tablesample (4 rows); + +explain +select index, array[index] from array_table; +select index, array[index] from array_table; + +create table map_table (data map, key string ); +insert into table map_table select map('k1','one','k2','two','k3','three'), concat('k', cast(key%3 as int)+1) from src tablesample (4 rows); + +explain +select key, data[key] from map_table; +select key, data[key] from map_table; diff --git ql/src/test/queries/negative/invalid_map_index2.q ql/src/test/queries/negative/invalid_map_index2.q deleted file mode 100644 index 5828f07..0000000 --- ql/src/test/queries/negative/invalid_map_index2.q +++ /dev/null @@ -1,2 +0,0 @@ -FROM src_thrift -INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint[0], src_thrift.mstringstring[concat('abc', 'abc')] diff --git ql/src/test/results/clientpositive/array_map_access_nonconstant.q.out ql/src/test/results/clientpositive/array_map_access_nonconstant.q.out new file mode 100644 index 0000000..dfb6cf1 --- /dev/null +++ ql/src/test/results/clientpositive/array_map_access_nonconstant.q.out @@ -0,0 +1,104 @@ +PREHOOK: query: create table array_table (array array, index int ) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table array_table (array array, index int ) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@array_table +PREHOOK: query: insert into table array_table select array('first', 'second', 'third'), key%3 from src tablesample (4 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@array_table +POSTHOOK: query: insert into table array_table select array('first', 'second', 'third'), key%3 from src tablesample (4 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@array_table +POSTHOOK: Lineage: array_table.array EXPRESSION [] +POSTHOOK: Lineage: array_table.index EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: explain +select index, array[index] from array_table +PREHOOK: type: QUERY +POSTHOOK: query: explain +select index, array[index] from array_table +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: array_table + Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: index (type: int), array[index] (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: select index, array[index] from array_table +PREHOOK: type: QUERY +PREHOOK: Input: default@array_table +#### A masked pattern was here #### +POSTHOOK: query: select index, array[index] from array_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@array_table +#### A masked pattern was here #### +1 second +2 third +2 third +0 first +PREHOOK: query: create table map_table (data map, key string ) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: create table map_table (data map, key string ) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@map_table +PREHOOK: query: insert into table map_table select map('k1','one','k2','two','k3','three'), concat('k', cast(key%3 as int)+1) from src tablesample (4 rows) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@map_table +POSTHOOK: query: insert into table map_table select map('k1','one','k2','two','k3','three'), concat('k', cast(key%3 as int)+1) from src tablesample (4 rows) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@map_table +POSTHOOK: Lineage: map_table.data EXPRESSION [] +POSTHOOK: Lineage: map_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: explain +select key, data[key] from map_table +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key, data[key] from map_table +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: map_table + Statistics: Num rows: 4 Data size: 100 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), data[key] (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 100 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: select key, data[key] from map_table +PREHOOK: type: QUERY +PREHOOK: Input: default@map_table +#### A masked pattern was here #### +POSTHOOK: query: select key, data[key] from map_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@map_table +#### A masked pattern was here #### +k2 two +k3 three +k3 three +k1 one diff --git ql/src/test/results/compiler/errors/invalid_map_index2.q.out ql/src/test/results/compiler/errors/invalid_map_index2.q.out deleted file mode 100644 index edc9bda..0000000 --- ql/src/test/results/compiler/errors/invalid_map_index2.q.out +++ /dev/null @@ -1,2 +0,0 @@ -Semantic Exception: -2:80 Non-constant expression for map indexes not supported. Error encountered near token ''abc'' \ No newline at end of file diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 5ccacf1..d717b38 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -47,6 +47,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -570,6 +572,13 @@ public static short getShort(Object o, PrimitiveObjectInspector oi) { return (short) getInt(o, oi); } + public static boolean isConvertibleToInteger(TypeInfo type) { + if (!(type instanceof PrimitiveTypeInfo)) { + return false; + } + PrimitiveCategory category = ((PrimitiveTypeInfo)type).getPrimitiveCategory(); + return category != PrimitiveCategory.BINARY && category != PrimitiveCategory.UNKNOWN; + } /** * Get the integer value out of a primitive object. Note that * NullPointerException will be thrown if o is null. Note that