[CARBONDATA-2409] Inappropriate Exception handling while select query with complex Data type

• Type: Bug
• Status: Open
• Priority: Major
• Resolution: Unresolved
• Affects Version/s: 1.4.0
• Component/s: data-query
• Environment: spark 2.2
• Assignee: Unassigned
• Reporter: Vandana Yadav
• Created: 27/Apr/18 11:20
• Updated: 27/Apr/18 11:20

Description

Inappropriate Exception handling while select query with complex Datatype

steps to Reproduce:

1) Create Table:

create table Array_com (CUST_ID string, YEAR int, MONTH int, AGE int, GENDER string, EDUCATED string, IS_MARRIED string, ARRAY_INT array<int>,ARRAY_STRING array<string>,ARRAY_DATE array<timestamp>,CARD_COUNT int,DEBIT_COUNT int, CREDIT_COUNT int, DEPOSIT double, HQ_DEPOSIT double) STORED BY 'org.apache.carbondata.format';

2) Load data into table:

LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/complex/Array.csv' INTO table Array_com options ('DELIMITER'=',', 'QUOTECHAR'='"', 'FILEHEADER'='CUST_ID,YEAR,MONTH,AGE,GENDER,EDUCATED,IS_MARRIED,ARRAY_INT,ARRAY_STRING,ARRAY_DATE,CARD_COUNT,DEBIT_COUNT,CREDIT_COUNT,DEPOSIT,HQ_DEPOSIT','COMPLEX_DELIMITER_LEVEL_1'='$');

3) Execute Query:

SELECT array_int[2], array_date[2],array_string[1], count(distinct array_date[2]) AS count_date,count(distinct array_string[2]) as count_string,count(distinct array_int[0]) as count_int FROM (select * from array_com) SUB_QRY GROUP BY array_int[2], array_date[2],array_string[1] order by array_int[2], array_date[2],array_string[1];

 

4) Expected Result: It should display the correct result as per the select query.

5) Actual Result:

Error: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 455.0 failed 1 times, most recent failure: Lost task 0.0 in stage 455.0 (TID 33517, localhost, executor driver): java.util.concurrent.ExecutionException: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 38, Column 25: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 38, Column 25: Cannot cast "org.apache.spark.unsafe.types.UTF8String" to "int"
/* 001 */ public java.lang.Object generate(Object[] references)

{ /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private UnsafeRow result;
/* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 011 */
/* 012 */ public SpecificUnsafeProjection(Object[] references) { /* 013 */ this.references = references; /* 014 */ result = new UnsafeRow(7); /* 015 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 64); /* 016 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 7); /* 017 */ /* 018 */ }
/* 019 */
/* 020 */ public void initialize(int partitionIndex) { /* 021 */ /* 022 */ }
/* 023 */
/* 024 */
/* 025 */ private void apply_1(InternalRow i) {
/* 026 */
/* 027 */
/* 028 */ boolean isNull6 = true;
/* 029 */ UTF8String value6 = null;
/* 030 */
/* 031 */ boolean isNull7 = i.isNullAt(1);
/* 032 */ ArrayData value7 = isNull7 ? null : (i.getArray(1));
/* 033 */ if (!isNull7) {
/* 034 */
/* 035 */
/* 036 */ isNull6 = false; // resultCode could change nullability.
/* 037 */
/* 038 */ final int index2 = (int) ((UTF8String) references[0]);
/* 039 */ if (index2 >= value7.numElements() || index2 < 0 || value7.isNullAt(index2)) { /* 040 */ isNull6 = true; /* 041 */ } else { /* 042 */ value6 = value7.getUTF8String(index2); /* 043 */ }
/* 044 */
/* 045 */
/* 046 */ }
/* 047 */ if (isNull6) { /* 048 */ rowWriter.setNullAt(2); /* 049 */ } else { /* 050 */ rowWriter.write(2, value6); /* 051 */ }
/* 052 */
/* 053 */
/* 054 */ boolean isNull9 = true;
/* 055 */ int value9 = -1;
/* 056 */
/* 057 */ boolean isNull10 = i.isNullAt(0);
/* 058 */ ArrayData value10 = isNull10 ? null : (i.getArray(0));
/* 059 */ if (!isNull10) {
/* 060 */
/* 061 */
/* 062 */ isNull9 = false; // resultCode could change nullability.
/* 063 */
/* 064 */ final int index3 = (int) 0;
/* 065 */ if (index3 >= value10.numElements() || index3 < 0 || value10.isNullAt(index3)) { /* 066 */ isNull9 = true; /* 067 */ } else { /* 068 */ value9 = value10.getInt(index3); /* 069 */ }
/* 070 */
/* 071 */
/* 072 */ }
/* 073 */ if (isNull9) { /* 074 */ rowWriter.setNullAt(3); /* 075 */ } else { /* 076 */ rowWriter.write(3, value9); /* 077 */ }
/* 078 */
/* 079 */ }
/* 080 */
/* 081 */
/* 082 */ private void apply_0(InternalRow i) {
/* 083 */
/* 084 */
/* 085 */ boolean isNull = true;
/* 086 */ int value = -1;
/* 087 */
/* 088 */ boolean isNull1 = i.isNullAt(0);
/* 089 */ ArrayData value1 = isNull1 ? null : (i.getArray(0));
/* 090 */ if (!isNull1) {
/* 091 */
/* 092 */
/* 093 */ isNull = false; // resultCode could change nullability.
/* 094 */
/* 095 */ final int index = (int) 2;
/* 096 */ if (index >= value1.numElements() || index < 0 || value1.isNullAt(index)) { /* 097 */ isNull = true; /* 098 */ } else { /* 099 */ value = value1.getInt(index); /* 100 */ }
/* 101 */
/* 102 */
/* 103 */ }
/* 104 */ if (isNull) { /* 105 */ rowWriter.setNullAt(0); /* 106 */ } else { /* 107 */ rowWriter.write(0, value); /* 108 */ }
/* 109 */
/* 110 */
/* 111 */ boolean isNull3 = true;
/* 112 */ long value3 = -1L;
/* 113 */
/* 114 */ boolean isNull4 = i.isNullAt(2);
/* 115 */ ArrayData value4 = isNull4 ? null : (i.getArray(2));
/* 116 */ if (!isNull4) {
/* 117 */
/* 118 */
/* 119 */ isNull3 = false; // resultCode could change nullability.
/* 120 */
/* 121 */ final int index1 = (int) 2L;
/* 122 */ if (index1 >= value4.numElements() || index1 < 0 || value4.isNullAt(index1)) { /* 123 */ isNull3 = true; /* 124 */ } else { /* 125 */ value3 = value4.getLong(index1); /* 126 */ }
/* 127 */
/* 128 */
/* 129 */ }
/* 130 */ if (isNull3) { /* 131 */ rowWriter.setNullAt(1); /* 132 */ } else { /* 133 */ rowWriter.write(1, value3); /* 134 */ }
/* 135 */
/* 136 */ }
/* 137 */
/* 138 */
/* 139 */ private void apply_2(InternalRow i) {
/* 140 */
/* 141 */
/* 142 */ final long value12 = -1L;
/* 143 */ if (true) { /* 144 */ rowWriter.setNullAt(4); /* 145 */ } else { /* 146 */ rowWriter.write(4, value12); /* 147 */ }
/* 148 */
/* 149 */
/* 150 */ final UTF8String value13 = null;
/* 151 */ if (true) { /* 152 */ rowWriter.setNullAt(5); /* 153 */ } else { /* 154 */ rowWriter.write(5, value13); /* 155 */ }
/* 156 */
/* 157 */
/* 158 */ rowWriter.write(6, 1);
/* 159 */
/* 160 */ }
/* 161 */
/* 162 */
/* 163 */ // Scala.Function1 need this
/* 164 */ public java.lang.Object apply(java.lang.Object row) { /* 165 */ return apply((InternalRow) row); /* 166 */ }
/* 167 */
/* 168 */ public UnsafeRow apply(InternalRow i) { /* 169 */ holder.reset(); /* 170 */ /* 171 */ rowWriter.zeroOutNullBytes(); /* 172 */ apply_0(i); /* 173 */ apply_1(i); /* 174 */ apply_2(i); /* 175 */ result.setTotalSize(holder.totalSize()); /* 176 */ return result; /* 177 */ }
/* 178 */ }

at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306)
at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293)
at org.spark_project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
at org.spark_project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135)
at org.spark_project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410)
at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380)
at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:946)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:412)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:366)
at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:930)
at org.apache.spark.sql.catalyst.expressions.UnsafeProjection$.create(Projection.scala:130)
at org.apache.spark.sql.catalyst.expressions.UnsafeProjection$.create(Projection.scala:140)
at org.apache.spark.sql.execution.ExpandExec$$anonfun$2.apply(ExpandExec.scala:53)
at org.apache.spark.sql.execution.ExpandExec$$anonfun$2.apply(ExpandExec.scala:53)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at org.apache.spark.sql.execution.ExpandExec$$anonfun$doExecute$1$$anonfun$3.apply(ExpandExec.scala:59)
at org.apache.spark.sql.execution.ExpandExec$$anonfun$doExecute$1$$anonfun$3.apply(ExpandExec.scala:58)
at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:797)
at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:797)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)
at org.apache.spark.scheduler.Task.run(Task.scala:108)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:338)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 38, Column 25: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 38, Column 25: Cannot cast "org.apache.spark.unsafe.types.UTF8String" to "int"
/* 001 */ public java.lang.Object generate(Object[] references) { /* 002 */ return new SpecificUnsafeProjection(references); /* 003 */ }

/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private UnsafeRow result;
/* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 011 */
/* 012 */ public SpecificUnsafeProjection(Object[] references)

{ /* 013 */ this.references = references; /* 014 */ result = new UnsafeRow(7); /* 015 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 64); /* 016 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 7); /* 017 */ /* 018 */ }

/* 019 */
/* 020 */ public void initialize(int partitionIndex)

{ /* 021 */ /* 022 */ }

/* 023 */
/* 024 */
/* 025 */ private void apply_1(InternalRow i) {
/* 026 */
/* 027 */
/* 028 */ boolean isNull6 = true;
/* 029 */ UTF8String value6 = null;
/* 030 */
/* 031 */ boolean isNull7 = i.isNullAt(1);
/* 032 */ ArrayData value7 = isNull7 ? null : (i.getArray(1));
/* 033 */ if (!isNull7) {
/* 034 */
/* 035 */
/* 036 */ isNull6 = false; // resultCode could change nullability.
/* 037 */
/* 038 */ final int index2 = (int) ((UTF8String) references[0]);
/* 039 */ if (index2 >= value7.numElements() || index2 < 0 || value7.isNullAt(index2))

{ /* 040 */ isNull6 = true; /* 041 */ }

else

{ /* 042 */ value6 = value7.getUTF8String(index2); /* 043 */ }

/* 044 */
/* 045 */
/* 046 */ }
/* 047 */ if (isNull6)

{ /* 048 */ rowWriter.setNullAt(2); /* 049 */ }

else

{ /* 050 */ rowWriter.write(2, value6); /* 051 */ }

/* 052 */
/* 053 */
/* 054 */ boolean isNull9 = true;
/* 055 */ int value9 = -1;
/* 056 */
/* 057 */ boolean isNull10 = i.isNullAt(0);
/* 058 */ ArrayData value10 = isNull10 ? null : (i.getArray(0));
/* 059 */ if (!isNull10) {
/* 060 */
/* 061 */
/* 062 */ isNull9 = false; // resultCode could change nullability.
/* 063 */
/* 064 */ final int index3 = (int) 0;
/* 065 */ if (index3 >= value10.numElements() || index3 < 0 || value10.isNullAt(index3))

{ /* 066 */ isNull9 = true; /* 067 */ }

else

{ /* 068 */ value9 = value10.getInt(index3); /* 069 */ }

/* 070 */
/* 071 */
/* 072 */ }
/* 073 */ if (isNull9)

{ /* 074 */ rowWriter.setNullAt(3); /* 075 */ }

else

{ /* 076 */ rowWriter.write(3, value9); /* 077 */ }

/* 078 */
/* 079 */ }
/* 080 */
/* 081 */
/* 082 */ private void apply_0(InternalRow i) {
/* 083 */
/* 084 */
/* 085 */ boolean isNull = true;
/* 086 */ int value = -1;
/* 087 */
/* 088 */ boolean isNull1 = i.isNullAt(0);
/* 089 */ ArrayData value1 = isNull1 ? null : (i.getArray(0));
/* 090 */ if (!isNull1) {
/* 091 */
/* 092 */
/* 093 */ isNull = false; // resultCode could change nullability.
/* 094 */
/* 095 */ final int index = (int) 2;
/* 096 */ if (index >= value1.numElements() || index < 0 || value1.isNullAt(index))

{ /* 097 */ isNull = true; /* 098 */ }

else

{ /* 099 */ value = value1.getInt(index); /* 100 */ }

/* 101 */
/* 102 */
/* 103 */ }
/* 104 */ if (isNull)

{ /* 105 */ rowWriter.setNullAt(0); /* 106 */ }

else

{ /* 107 */ rowWriter.write(0, value); /* 108 */ }

/* 109 */
/* 110 */
/* 111 */ boolean isNull3 = true;
/* 112 */ long value3 = -1L;
/* 113 */
/* 114 */ boolean isNull4 = i.isNullAt(2);
/* 115 */ ArrayData value4 = isNull4 ? null : (i.getArray(2));
/* 116 */ if (!isNull4) {
/* 117 */
/* 118 */
/* 119 */ isNull3 = false; // resultCode could change nullability.
/* 120 */
/* 121 */ final int index1 = (int) 2L;
/* 122 */ if (index1 >= value4.numElements() || index1 < 0 || value4.isNullAt(index1))

{ /* 123 */ isNull3 = true; /* 124 */ }

else

{ /* 125 */ value3 = value4.getLong(index1); /* 126 */ }

/* 127 */
/* 128 */
/* 129 */ }
/* 130 */ if (isNull3)

{ /* 131 */ rowWriter.setNullAt(1); /* 132 */ }

else

{ /* 133 */ rowWriter.write(1, value3); /* 134 */ }

/* 135 */
/* 136 */ }
/* 137 */
/* 138 */
/* 139 */ private void apply_2(InternalRow i) {
/* 140 */
/* 141 */
/* 142 */ final long value12 = -1L;
/* 143 */ if (true)

{ /* 144 */ rowWriter.setNullAt(4); /* 145 */ }

else

{ /* 146 */ rowWriter.write(4, value12); /* 147 */ }

/* 148 */
/* 149 */
/* 150 */ final UTF8String value13 = null;
/* 151 */ if (true)

{ /* 152 */ rowWriter.setNullAt(5); /* 153 */ }

else

{ /* 154 */ rowWriter.write(5, value13); /* 155 */ }

/* 156 */
/* 157 */
/* 158 */ rowWriter.write(6, 1);
/* 159 */
/* 160 */ }
/* 161 */
/* 162 */
/* 163 */ // Scala.Function1 need this
/* 164 */ public java.lang.Object apply(java.lang.Object row)

{ /* 165 */ return apply((InternalRow) row); /* 166 */ }

/* 167 */
/* 168 */ public UnsafeRow apply(InternalRow i)

{ /* 169 */ holder.reset(); /* 170 */ /* 171 */ rowWriter.zeroOutNullBytes(); /* 172 */ apply_0(i); /* 173 */ apply_1(i); /* 174 */ apply_2(i); /* 175 */ result.setTotalSize(holder.totalSize()); /* 176 */ return result; /* 177 */ }

/* 178 */ }

at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1010)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1067)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1064)
at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
... 39 more

Driver stacktrace: (state=,code=0)