Details
-
Bug
-
Status: Resolved
-
Major
-
Resolution: Fixed
-
2.4.0, 3.0.0
-
None
Description
Inporgress applications with last updated time is withing the cleaning interval are getting deleted.
Added a UT to test the scenario.
test("should not clean inprogress application with lastUpdated time less the maxTime") { val firstFileModifiedTime = TimeUnit.DAYS.toMillis(1) val secondFileModifiedTime = TimeUnit.DAYS.toMillis(6) val maxAge = TimeUnit.DAYS.toMillis(7) val clock = new ManualClock(0) val provider = new FsHistoryProvider( createTestConf().set("spark.history.fs.cleaner.maxAge", s"${maxAge}ms"), clock) val log = newLogFile("inProgressApp1", None, inProgress = true) writeFile(log, true, None, SparkListenerApplicationStart( "inProgressApp1", Some("inProgressApp1"), 3L, "test", Some("attempt1")) ) clock.setTime(firstFileModifiedTime) provider.checkForLogs() writeFile(log, true, None, SparkListenerApplicationStart( "inProgressApp1", Some("inProgressApp1"), 3L, "test", Some("attempt1")), SparkListenerJobStart(0, 1L, Nil, null) ) clock.setTime(secondFileModifiedTime) provider.checkForLogs() clock.setTime(TimeUnit.DAYS.toMillis(10)) writeFile(log, true, None, SparkListenerApplicationStart( "inProgressApp1", Some("inProgressApp1"), 3L, "test", Some("attempt1")), SparkListenerJobStart(0, 1L, Nil, null), SparkListenerJobEnd(0, 1L, JobSucceeded) ) provider.checkForLogs() // This should not trigger any cleanup updateAndCheck(provider) { list => list.size should be(1) } }