[SPARK-20934] Task is hung at inner join, would work with other kind of joins

• Type: Bug
• Status: Resolved
• Priority: Major
• Resolution: Invalid
• Affects Version/s: 2.1.0
• Component/s: Shuffle, Spark Core
• Assignee: Unassigned
• Reporter: Mohamed Elagamy
• Created: 31/May/17 06:41
• Updated: 31/May/17 08:24
• Resolved: 31/May/17 08:24

Description

I am using spark 2.1.0 to read from parquets and inner join between different dataframes, but it gets stuck at the inner join step and never show any progress, here is the thread dump

sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:208)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:169)
scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:169)
scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
scala.concurrent.Await$.ready(package.scala:169)
com.wd.perf.collector.spark.report.utils.GroupUtils$.generateReportInGroups(GroupUtils.scala:70)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$.com$wd$perf$collector$spark$report$jobs$definition$JobSummaryDefinitionParquetCreator$$generateHourlyBySWH(JobSummaryDefinitionParquetCreator.scala:82)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1$$anonfun$apply$1.apply(JobSummaryDefinitionParquetCreator.scala:40)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1$$anonfun$apply$1.apply(JobSummaryDefinitionParquetCreator.scala:39)
scala.collection.immutable.List.foreach(List.scala:381)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1.apply(JobSummaryDefinitionParquetCreator.scala:39)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1.apply(JobSummaryDefinitionParquetCreator.scala:38)
com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$$anonfun$5.apply(ParquetReportInstaller.scala:148)
com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$$anonfun$5.apply(ParquetReportInstaller.scala:110)
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
scala.collection.immutable.List.foreach(List.scala:381)
scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
scala.collection.immutable.List.map(List.scala:285)
com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$.executeReportStep(ParquetReportInstaller.scala:110)
com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$.executeByInstallOption(ParquetReportInstaller.scala:51)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$.generateReport(JobSummaryDefinitionParquetCreator.scala:44)
com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor.com$wd$perf$collector$spark$actor$report$parquet$ReportNameInstanceCreatorActor$$processMessage(ReportNameInstanceCreatorActor.scala:67)
com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor$$anonfun$receive$1$$anonfun$1$$anonfun$apply$1.apply$mcV$sp(ReportNameInstanceCreatorActor.scala:36)
com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor$$anonfun$receive$1$$anonfun$1$$anonfun$apply$1.apply(ReportNameInstanceCreatorActor.scala:35)
com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor$$anonfun$receive$1$$anonfun$1$$anonfun$apply$1.apply(ReportNameInstanceCreatorActor.scala:35)
scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)
scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)
akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:39)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)