[SPARK-10155] Memory leak in SQL parsers - ASF JIRA

XML

Word

Printable

JSON

Details

Type: Bug
Status: Resolved
Priority: Critical
Resolution: Fixed
Affects Version/s: None
Fix Version/s: 1.5.1, 1.6.0
Component/s: SQL
Labels:
None

Target Version/s:

1.5.1, 1.6.0

Description

I saw a lot of `ThreadLocal` objects in the following app:

import org.apache.spark._
import org.apache.spark.sql._

object SparkApp {

  def foo(sqlContext: SQLContext): Unit = {
    import sqlContext.implicits._
    sqlContext.sparkContext.parallelize(Seq("aaa", "bbb", "ccc")).toDF().filter("length(_1) > 0").count()
  }

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("sql-memory-leak")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)
    while (true) {
      foo(sqlContext)
    }
  }
}