diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java
index 1079d29..611c154 100644
--- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java
+++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java
@@ -28,6 +28,7 @@ import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.JUnitCore;
 import org.junit.runner.Result;
+import org.junit.runner.notification.Failure;
 
 public class TestFailIfDirectoryNotClosed extends WithNestedTests {
   public TestFailIfDirectoryNotClosed() {
@@ -45,34 +46,51 @@ public class TestFailIfDirectoryNotClosed extends WithNestedTests {
     public void testDummy() throws IOException {
       MockDirectoryWrapper dir = newMockDirectory();
       IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
-      dir.close();
+      try {
+        dir.close(); // will throw an exception.
+      } finally {
+        iw.close();  // close the writer to allow directory cleanup.
+      }
     }
   }
-  
+
   public static class Nested3 extends WithNestedTests.AbstractNestedTest {
     public void testDummy() throws IOException {
       MockDirectoryWrapper dir = newMockDirectory();
       dir.setLockFactory(new SingleInstanceLockFactory());
       IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
-      dir.close();
+      try {
+        dir.close(); // will throw an exception.
+      } finally {
+        iw.close();  // close the writer to allow directory cleanup.
+      }
     }
   }
 
   @Test
   public void testFailIfDirectoryNotClosed() {
     Result r = JUnitCore.runClasses(Nested1.class);
-    Assert.assertEquals(1, r.getFailureCount());
+    assertContainsFailureWith(r, "java.lang.AssertionError: Directory not closed:");
   }
-  
+
   @Test
   public void testFailIfIndexWriterNotClosed() {
     Result r = JUnitCore.runClasses(Nested2.class);
-    Assert.assertEquals(1, r.getFailureCount());
+    assertContainsFailureWith(r, "there are still open locks");
   }
-  
+
   @Test
   public void testFailIfIndexWriterNotClosedChangeLockFactory() {
     Result r = JUnitCore.runClasses(Nested3.class);
-    Assert.assertEquals(1, r.getFailureCount());
+    assertContainsFailureWith(r, "there are still open locks");
+  }
+  
+  private void assertContainsFailureWith(Result r, String msg) {
+    for (Failure f : r.getFailures()) {
+      if (f.getTrace().contains(msg)) {
+        return;
+      }
+    }
+    Assert.assertTrue("No failure with message: " + msg, false);
   }
 }
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/CloseableDirectory.java b/lucene/test-framework/src/java/org/apache/lucene/util/CloseableDirectory.java
index 57a2132..8d0e85e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/CloseableDirectory.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/CloseableDirectory.java
@@ -3,7 +3,6 @@ package org.apache.lucene.util;
 import java.io.Closeable;
 
 import org.apache.lucene.store.BaseDirectoryWrapper;
-import org.apache.lucene.store.MockDirectoryWrapper;
 import org.junit.Assert;
 
 /*
@@ -31,9 +30,8 @@ import org.junit.Assert;
 final class CloseableDirectory implements Closeable {
   private final BaseDirectoryWrapper dir;
   private final TestRuleMarkFailure failureMarker;
-  
-  public CloseableDirectory(BaseDirectoryWrapper dir,
-      TestRuleMarkFailure failureMarker) {
+
+  public CloseableDirectory(BaseDirectoryWrapper dir, TestRuleMarkFailure failureMarker) {
     this.dir = dir;
     this.failureMarker = failureMarker;
   }