Major Description
After recent changes, Hive on Spark throws KryoException:
16/04/14 21:53:24 INFO hiveserver2.DDLTest: ERROR : Failed to execute spark task, with exception 'java.lang.Exception(Failed to submit Spark work, please retry later)' 16/04/14 21:53:24 INFO hiveserver2.DDLTest: java.lang.Exception: Failed to submit Spark work, please retry later 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.exec.spark.RemoteHiveSparkClient.execute(RemoteHiveSparkClient.java:174) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionImpl.submit(SparkSessionImpl.java:71) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.exec.spark.SparkTask.execute(SparkTask.java:103) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:100) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1769) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1526) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1305) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1114) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1107) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:178) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.service.cli.operation.SQLOperation.access$100(SQLOperation.java:72) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.service.cli.operation.SQLOperation$2$1.run(SQLOperation.java:232) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at java.security.AccessController.doPrivileged(Native Method) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at javax.security.auth.Subject.doAs(Subject.java:415) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1693) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.service.cli.operation.SQLOperation$2.run(SQLOperation.java:245) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at java.util.concurrent.FutureTask.run(FutureTask.java:262) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at java.lang.Thread.run(Thread.java:745) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: Caused by: org.apache.hive.com.esotericsoftware.kryo.KryoException: java.util.ConcurrentModificationException 16/04/14 21:53:24 INFO hiveserver2.DDLTest: Serialization trace: 16/04/14 21:53:24 INFO hiveserver2.DDLTest: classes (sun.misc.Launcher$AppClassLoader) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: classloader (java.security.ProtectionDomain) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: context (java.security.AccessControlContext) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: acc (java.net.URLClassLoader) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: classLoader (org.apache.hadoop.hive.conf.HiveConf) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: conf (org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: metrics (org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics$CodahaleMetricsScope) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: openScopes (org.apache.hadoop.hive.ql.log.PerfLogger) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: perfLogger (org.apache.hadoop.hive.ql.exec.SparkHashTableSinkOperator) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: childOperators (org.apache.hadoop.hive.ql.exec.TableScanOperator) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: aliasToWork (org.apache.hadoop.hive.ql.plan.MapWork) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: invertedWorkGraph (org.apache.hadoop.hive.ql.plan.SparkWork) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:82) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:614) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:337) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:276) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObjectOrNull(Kryo.java:592) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:68) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:614) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.write(MapSerializer.java:91) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.write(MapSerializer.java:17) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:614) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.CollectionSerializer.write(CollectionSerializer.java:78) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.CollectionSerializer.write(CollectionSerializer.java:18) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:614) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.write(MapSerializer.java:91) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.write(MapSerializer.java:17) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:614) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.write(MapSerializer.java:84) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.MapSerializer.write(MapSerializer.java:17) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:474) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:520) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.exec.spark.KryoSerializer.serialize(KryoSerializer.java:42) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.exec.spark.RemoteHiveSparkClient.submit(RemoteHiveSparkClient.java:191) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hadoop.hive.ql.exec.spark.RemoteHiveSparkClient.execute(RemoteHiveSparkClient.java:172) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: ... 21 more 16/04/14 21:53:24 INFO hiveserver2.DDLTest: Caused by: java.util.ConcurrentModificationException 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at java.util.Vector$Itr.checkForComodification(Vector.java:1156) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at java.util.Vector$Itr.next(Vector.java:1133) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.CollectionSerializer.write(CollectionSerializer.java:70) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.CollectionSerializer.write(CollectionSerializer.java:18) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:538) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: at org.apache.hive.com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:61) 16/04/14 21:53:24 INFO hiveserver2.DDLTest: ... 74 more 16/04/14 21:53:24 INFO hiveserver2.DDLTest: 16/04/14 21:53:24 INFO hiveserver2.DDLTest: ERROR : Failed to execute spark task, with exception 'java.lang.Exception(Failed to submit Spark work, please retry later)' 16/04/14 21:53:24 INFO hiveserver2.DDLTest: java.lang.Exception: Failed to submit Spark work, please retry later