Major Description
This example properly fires the event
spark.range(100)
.observe(
name = "other_event",
avg($"id").cast("int").as("avg_val"))
.collect()
But when I add persist, then no event is fired or seen (not sure which):
spark.range(100)
.observe(
name = "my_event",
avg($"id").cast("int").as("avg_val"))
.persist()
.collect()
The listener:
val metricMaps = ArrayBuffer.empty[Map[String, Row]]
val listener = new QueryExecutionListener {
override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
metricMaps += qe.observedMetrics
}
override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
// No-op
}
}
spark.listenerManager.register(listener)
