Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/data/GarbageCollector.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/data/GarbageCollector.java (revision 1461410) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/data/GarbageCollector.java (working copy) @@ -18,6 +18,7 @@ import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; import org.apache.jackrabbit.api.management.MarkEventListener; +import org.apache.jackrabbit.core.RepositoryContext; import org.apache.jackrabbit.core.SessionImpl; import org.apache.jackrabbit.core.id.NodeId; import org.apache.jackrabbit.core.id.PropertyId; @@ -96,6 +97,8 @@ private final SessionImpl[] sessionList; private final AtomicBoolean closed = new AtomicBoolean(); + + private final RepositoryContext context; private boolean persistenceManagerScan; @@ -105,14 +108,17 @@ * Create a new garbage collector. * This method is usually not called by the application, it is called * by SessionImpl.createDataStoreGarbageCollector(). - * + * + * @param context repository context * @param dataStore the data store to be garbage-collected * @param list the persistence managers * @param sessionList the sessions to access the workspaces */ - public GarbageCollector( + + public GarbageCollector( RepositoryContext context, DataStore dataStore, IterablePersistenceManager[] list, SessionImpl[] sessionList) { + this.context = context; this.store = dataStore; this.pmList = list; this.persistenceManagerScan = list != null; @@ -235,6 +241,7 @@ * Stop the observation listener if any are installed. */ public void stopScan() throws RepositoryException { + context.setGcRunning(false); if (listeners.size() > 0) { for (Listener listener : listeners) { listener.stop(); Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/RepositoryContext.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/RepositoryContext.java (revision 1461410) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/RepositoryContext.java (working copy) @@ -122,6 +122,12 @@ * The Statistics manager, handles statistics */ private StatManager statManager; + + /** + * boolean flag to indicate if GC is running + */ + + private boolean gcRunning; /** * Creates a component context for the given repository. @@ -429,4 +435,14 @@ return statManager; } + public boolean isGcRunning() { + return gcRunning; + } + + public synchronized void setGcRunning(boolean gcRunning) { + this.gcRunning = gcRunning; + } + + + } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/RepositoryImpl.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/RepositoryImpl.java (revision 1461410) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/RepositoryImpl.java (working copy) @@ -52,6 +52,7 @@ import org.apache.commons.collections.map.ReferenceMap; import org.apache.commons.io.IOUtils; import org.apache.jackrabbit.api.JackrabbitRepository; +import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; import org.apache.jackrabbit.api.management.RepositoryManager; import org.apache.jackrabbit.api.security.authentication.token.TokenCredentials; import org.apache.jackrabbit.commons.AbstractRepository; @@ -1384,7 +1385,7 @@ * to access this functionality. This RepositoryImpl method may be * removed in future Jackrabbit versions. */ - public GarbageCollector createDataStoreGarbageCollector() + public GarbageCollector createDataStoreGarbageCollector() throws RepositoryException { ArrayList pmList = new ArrayList(); InternalVersionManagerImpl vm = context.getInternalVersionManager(); @@ -1422,7 +1423,14 @@ } ipmList[i] = (IterablePersistenceManager) pm; } - return new GarbageCollector(context.getDataStore(), ipmList, sessions); + GarbageCollector gc = new GarbageCollector(context, context.getDataStore(), ipmList, sessions); + synchronized (this) { + if (context.isGcRunning()) { + throw new RepositoryException("Cannot create GC. GC already running"); + } + context.setGcRunning(true); + } + return gc; } //-----------------------------------------------------------< Repository > Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/ConcurrentGcTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/ConcurrentGcTest.java (revision 1461410) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/ConcurrentGcTest.java (working copy) @@ -1,190 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.core.data; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.Random; -import java.util.Set; -import junit.framework.TestCase; -import org.apache.commons.io.FileUtils; -import org.apache.jackrabbit.core.data.db.DbDataStore; -import org.apache.jackrabbit.core.util.db.ConnectionFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Tests concurrent garbage collection, see JCR-2026 - */ -public class ConcurrentGcTest extends TestCase { - - static final Logger LOG = LoggerFactory.getLogger(ConcurrentGcTest.class); - - private static final String TEST_DIR = "target/ConcurrentGcTest"; - - protected DataStore store; - private Thread gcLoopThread; - - protected Set ids = Collections.synchronizedSet(new HashSet()); - - protected volatile boolean gcLoopStop; - protected volatile Exception gcException; - - public void setUp() throws IOException { - deleteAll(); - } - - public void tearDown() throws IOException { - deleteAll(); - } - - private void deleteAll() throws IOException { - FileUtils.deleteDirectory(new File(TEST_DIR)); - } - - public void testDatabases() throws Exception { -// doTestDatabase( -// "org.h2.Driver", -// "jdbc:h2:" + TEST_DIR + "/db", -// "sa", "sa"); - - // not enabled by default - // doTestDatabase( - // "org.postgresql.Driver", - // "jdbc:postgresql:test", - // "sa", "sa"); - - // not enabled by default - // doTestDatabase( - // "com.mysql.jdbc.Driver", - // "jdbc:postgresql:test", - // "sa", "sa"); - - // fails with a deadlock - // doTestDatabase( - // "org.apache.derby.jdbc.EmbeddedDriver", - // "jdbc:derby:" + TEST_DIR + "/db;create=true", - // "sa", "sa"); - } - - private void doTestDatabase(String driver, String url, String user, String password) throws Exception { - ConnectionFactory pool = new ConnectionFactory(); - try { - DbDataStore store = new DbDataStore(); - store.setConnectionFactory(pool); - - ids.clear(); - - store.setDriver(driver); - store.setUrl(url); - store.setUser(user); - store.setPassword(password); - - store.init("target/test-db-datastore"); - store.setMinRecordLength(0); - doTest(store); - } finally { - pool.close(); - } - } - - public void testFile() throws Exception { - FileDataStore store = new FileDataStore(); - store.setPath(TEST_DIR + "/fs"); - store.init(TEST_DIR + "/fs"); - store.setMinRecordLength(0); - doTest(store); - } - - void doTest(DataStore store) throws Exception { - this.store = store; - - Random r = new Random(); - - concurrentGcLoopStart(); - - int len = 100; - if (getTestScale() > 1) { - len = 1000; - } - - for (int i = 0; i < len && gcException == null; i++) { - LOG.info("test " + i); - byte[] data = new byte[3]; - r.nextBytes(data); - DataRecord rec = store.addRecord(new ByteArrayInputStream(data)); - LOG.debug(" added " + rec.getIdentifier()); - if (r.nextBoolean()) { - LOG.debug(" added " + rec.getIdentifier() + " -> keep reference"); - ids.add(rec.getIdentifier()); - store.getRecord(rec.getIdentifier()); - } - if (r.nextInt(100) == 0) { - LOG.debug("clear i: " + i); - ids.clear(); - } - } - concurrentGcLoopStop(); - store.close(); - } - - private void concurrentGcLoopStart() { - gcLoopStop = false; - gcException = null; - - gcLoopThread = new Thread() { - public void run() { - try { - while (!gcLoopStop) { - if (ids.size() > 0) { - // store.clearInUse(); - long now = System.currentTimeMillis(); - LOG.debug("gc now: " + now); - store.updateModifiedDateOnAccess(now); - for (DataIdentifier id : new ArrayList(ids)) { - LOG.debug(" gc touch " + id); - store.getRecord(id); - } - int count = store.deleteAllOlderThan(now); - LOG.debug("gc now: " + now + " done, deleted: " + count); - } - } - } catch (DataStoreException e) { - gcException = e; - } - } - }; - gcLoopThread.start(); - } - - private void concurrentGcLoopStop() throws Exception { - gcLoopStop = true; - gcLoopThread.join(); - if (gcException != null) { - throw gcException; - } - } - - static int getTestScale() { - return Integer.parseInt(System.getProperty("jackrabbit.test.scale", "1")); - } - -} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GarbageCollectorTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GarbageCollectorTest.java (revision 1461410) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GarbageCollectorTest.java (working copy) @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.core.data; - -import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; -import org.apache.jackrabbit.api.management.MarkEventListener; -import org.apache.jackrabbit.core.SessionImpl; -import org.apache.jackrabbit.test.AbstractJCRTest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import EDU.oswego.cs.dl.util.concurrent.SynchronousChannel; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Iterator; -import javax.jcr.Credentials; -import javax.jcr.Node; -import javax.jcr.NodeIterator; -import javax.jcr.RepositoryException; -import javax.jcr.Session; -import javax.jcr.ValueFactory; - -/** - * Test cases for data store garbage collection. - */ -public class GarbageCollectorTest extends AbstractJCRTest implements ScanEventListener { - - /** logger instance */ - private static final Logger LOG = LoggerFactory.getLogger(GarbageCollectorTest.class); - - public void testCloseSessionWhileRunningGc() throws Exception { - final Session session = getHelper().getReadWriteSession(); - - final DataStoreGarbageCollector gc = ((SessionImpl) session).createDataStoreGarbageCollector(); - final Exception[] ex = new Exception[1]; - gc.setMarkEventListener(new MarkEventListener() { - boolean closed; - - public void beforeScanning(Node n) throws RepositoryException { - closeTest(); - } - - private void closeTest() throws RepositoryException { - if (closed) { - ex[0] = new Exception("Scanning after the session is closed"); - } - closed = true; - session.logout(); - } - - }); - try { - gc.mark(); - fail("Exception 'session has been closed' expected"); - } catch (RepositoryException e) { - LOG.debug("Expected exception caught: " + e.getMessage()); - } - if (ex[0] != null) { - throw ex[0]; - } - gc.close(); - } - - public void testConcurrentGC() throws Exception { - Node root = testRootNode; - Session session = root.getSession(); - - final SynchronousChannel sync = new SynchronousChannel(); - final Node node = root.addNode("slowBlob"); - final int blobLength = 1000; - final ValueFactory vf = session.getValueFactory(); - new Thread() { - public void run() { - try { - node.setProperty("slowBlob", vf.createBinary(new InputStream() { - int pos; - public int read() throws IOException { - pos++; - if (pos < blobLength) { - return pos % 80 == 0 ? '\n' : '.'; - } else if (pos == blobLength) { - try { - sync.put("x"); - // deleted - sync.take(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return 'x'; - } - return -1; - } - })); - node.getSession().save(); - sync.put("saved"); - } catch (Exception e) { - e.printStackTrace(); - } - } - }.start(); - assertEquals("x", sync.take()); - DataStoreGarbageCollector gc = ((SessionImpl) session).createDataStoreGarbageCollector(); - gc.mark(); - gc.sweep(); - sync.put("deleted"); - assertEquals("saved", sync.take()); - InputStream in = node.getProperty("slowBlob").getBinary().getStream(); - for (int pos = 1; pos < blobLength; pos++) { - int expected = pos % 80 == 0 ? '\n' : '.'; - assertEquals(expected, in.read()); - } - assertEquals('x', in.read()); - in.close(); - gc.close(); - } - - public void testGC() throws Exception { - Node root = testRootNode; - Session session = root.getSession(); - - deleteMyNodes(); - runGC(session, true); - - root.addNode("node1"); - Node node2 = root.addNode("node2"); - Node n = node2.addNode("nodeWithBlob"); - ValueFactory vf = session.getValueFactory(); - n.setProperty("test", vf.createBinary(new RandomInputStream(10, 1000))); - n = node2.addNode("nodeWithTemporaryBlob"); - n.setProperty("test", vf.createBinary(new RandomInputStream(11, 1000))); - session.save(); - - n.remove(); - session.save(); - - GarbageCollector gc = ((SessionImpl)session).createDataStoreGarbageCollector(); - - if (gc.getDataStore() instanceof FileDataStore) { - // make sure the file is old (access time resolution is 2 seconds) - Thread.sleep(2000); - } - - LOG.debug("scanning..."); - gc.mark(); - int count = listIdentifiers(gc); - LOG.debug("stop scanning; currently " + count + " identifiers"); - gc.stopScan(); - LOG.debug("deleting..."); - gc.getDataStore().clearInUse(); - assertTrue(gc.sweep() > 0); - int count2 = listIdentifiers(gc); - assertEquals(count - 1, count2); - - deleteMyNodes(); - - gc.close(); - } - - private void runGC(Session session, boolean all) throws Exception { - GarbageCollector gc = ((SessionImpl)session).createDataStoreGarbageCollector(); - gc.setMarkEventListener(this); - if (gc.getDataStore() instanceof FileDataStore) { - // make sure the file is old (access time resolution is 2 seconds) - Thread.sleep(2000); - } - gc.mark(); - gc.stopScan(); - if (all) { - gc.getDataStore().clearInUse(); - } - gc.sweep(); - gc.close(); - } - - private int listIdentifiers(GarbageCollector gc) throws DataStoreException { - LOG.debug("identifiers:"); - int count = 0; - Iterator it = gc.getDataStore().getAllIdentifiers(); - while (it.hasNext()) { - DataIdentifier id = it.next(); - LOG.debug(" " + id); - count++; - } - return count; - } - - public void testTransientObjects() throws Exception { - - Node root = testRootNode; - Session session = root.getSession(); - - deleteMyNodes(); - - Credentials cred = getHelper().getSuperuserCredentials(); - Session s2 = getHelper().getRepository().login(cred); - root = s2.getRootNode(); - Node node2 = root.addNode("node3"); - Node n = node2.addNode("nodeWithBlob"); - ValueFactory vf = session.getValueFactory(); - n.setProperty("test", vf.createBinary(new RandomInputStream(10, 1000))); - - runGC(session, false); - - s2.save(); - - InputStream in = n.getProperty("test").getBinary().getStream(); - InputStream in2 = new RandomInputStream(10, 1000); - while (true) { - int a = in.read(); - int b = in2.read(); - assertEquals(a, b); - if (a < 0) { - break; - } - } - - deleteMyNodes(); - - s2.logout(); - } - - public void afterScanning(Node n) throws RepositoryException { - if (n != null && n.getPath().startsWith("/testroot/node")) { - String path = n.getPath(); - LOG.debug("scanned: " + path); - } - } - - private void list(Node n) throws RepositoryException { - if (!n.getName().startsWith("jcr:")) { - for (NodeIterator it = n.getNodes(); it.hasNext();) { - list(it.nextNode()); - } - } - } - - public void beforeScanning(Node n) throws RepositoryException { - if (n != null && n.getPath().equals("/testroot/node2")) { - Session session = n.getSession(); - list(session.getRootNode()); - session.move("/testroot/node2/nodeWithBlob", "/testroot/node1/nodeWithBlob"); - session.save(); - LOG.debug("moved /testroot/node2/nodeWithBlob to /testroot/node1"); - } - } - - private void deleteMyNodes() throws RepositoryException { - Node root = testRootNode; - while (root.hasNode("testroot")) { - root.getNode("testroot").remove(); - } - root.getSession().save(); - } - -} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCConcurrentTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCConcurrentTest.java (revision 1461410) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCConcurrentTest.java (working copy) @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.core.data; - -import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; -import org.apache.jackrabbit.api.management.MarkEventListener; -import org.apache.jackrabbit.core.SessionImpl; -import org.apache.jackrabbit.test.AbstractJCRTest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Random; - -import javax.jcr.Node; -import javax.jcr.Property; -import javax.jcr.RepositoryException; -import javax.jcr.Session; -import javax.jcr.ValueFactory; - -/** - * Test case for concurrent garbage collection - */ -public class GCConcurrentTest extends AbstractJCRTest { - - /** logger instance */ - private static final Logger LOG = LoggerFactory.getLogger(GCConcurrentTest.class); - - public void testConcurrentDelete() throws Exception { - Node root = testRootNode; - Session session = root.getSession(); - - final String testNodeName = "testConcurrentDelete"; - node(root, testNodeName); - session.save(); - DataStoreGarbageCollector gc = ((SessionImpl) session).createDataStoreGarbageCollector(); - gc.setPersistenceManagerScan(false); - gc.setMarkEventListener(new MarkEventListener() { - public void beforeScanning(Node n) throws RepositoryException { - if (n.getName().equals(testNodeName)) { - n.remove(); - n.getSession().save(); - } - } - - }); - gc.mark(); - gc.close(); - } - - public void testGC() throws Exception { - Node root = testRootNode; - Session session = root.getSession(); - - GCThread gc = new GCThread(session); - Thread gcThread = new Thread(gc, "Datastore Garbage Collector"); - - int len = 10 * getTestScale(); - boolean started = false; - for (int i = 0; i < len; i++) { - if (!started && i > 5 + len / 100) { - started = true; - gcThread.start(); - } - Node n = node(root, "test" + i); - ValueFactory vf = session.getValueFactory(); - n.setProperty("data", vf.createBinary(randomInputStream(i))); - session.save(); - LOG.debug("saved: " + i); - } - Thread.sleep(10); - for (int i = 0; i < len; i++) { - Node n = root.getNode("test" + i); - Property p = n.getProperty("data"); - InputStream in = p.getBinary().getStream(); - InputStream expected = randomInputStream(i); - checkStreams(expected, in); - n.remove(); - LOG.debug("removed: " + i); - session.save(); - } - Thread.sleep(10); - gc.setStop(true); - Thread.sleep(10); - gcThread.join(); - gc.throwException(); - } - - private void checkStreams(InputStream expected, InputStream in) throws IOException { - while (true) { - int e = expected.read(); - int i = in.read(); - if (e < 0 || i < 0) { - if (e >= 0 || i >= 0) { - fail("expected: " + e + " got: " + i); - } - break; - } else { - assertEquals(e, i); - } - } - expected.close(); - in.close(); - } - - static InputStream randomInputStream(long seed) { - byte[] data = new byte[4096]; - new Random(seed).nextBytes(data); - return new ByteArrayInputStream(data); - } - - static Node node(Node n, String x) throws RepositoryException { - return n.hasNode(x) ? n.getNode(x) : n.addNode(x); - } - - static int getTestScale() { - return Integer.parseInt(System.getProperty("jackrabbit.test.scale", "1")); - } - -} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCEventListenerTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCEventListenerTest.java (revision 1461410) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCEventListenerTest.java (working copy) @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.core.data; - -import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; -import org.apache.jackrabbit.api.management.MarkEventListener; -import org.apache.jackrabbit.core.SessionImpl; -import org.apache.jackrabbit.test.AbstractJCRTest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.util.Random; - -import javax.jcr.Node; -import javax.jcr.RepositoryException; -import javax.jcr.Session; -import javax.jcr.ValueFactory; - -/** - * Test the DataStore garbage collector. - * This tests that the EventListener is called while scanning the repository. - * - * @author Thomas Mueller - */ -public class GCEventListenerTest extends AbstractJCRTest implements MarkEventListener { - - /** logger instance */ - private static final Logger LOG = LoggerFactory.getLogger(GCEventListenerTest.class); - - private static final String TEST_NODE_NAME = "testGCEventListener"; - - private boolean gotNullNode; - private boolean gotNode; - private int count; - - public void testEventListener() throws Exception { - doTestEventListener(true); - doTestEventListener(false); - } - - private void doTestEventListener(boolean allowPmScan) throws Exception { - Node root = testRootNode; - Session session = root.getSession(); - if (root.hasNode(TEST_NODE_NAME)) { - root.getNode(TEST_NODE_NAME).remove(); - session.save(); - } - Node test = root.addNode(TEST_NODE_NAME); - Random random = new Random(); - byte[] data = new byte[10000]; - for (int i = 0; i < 10; i++) { - Node n = test.addNode("x" + i); - random.nextBytes(data); - ValueFactory vf = session.getValueFactory(); - n.setProperty("data", vf.createBinary(new ByteArrayInputStream(data))); - session.save(); - if (i % 2 == 0) { - n.remove(); - session.save(); - } - } - session.save(); - SessionImpl si = (SessionImpl) session; - DataStoreGarbageCollector gc = si.createDataStoreGarbageCollector(); - DataStore ds = ((GarbageCollector) gc).getDataStore(); - if (ds != null) { - ds.clearInUse(); - boolean pmScan = gc.isPersistenceManagerScan(); - gc.setPersistenceManagerScan(allowPmScan); - gotNullNode = false; - gotNode = false; - gc.setMarkEventListener(this); - gc.mark(); - if (pmScan && allowPmScan) { - assertTrue("PM scan without null Node", gotNullNode); - assertFalse("PM scan, but got a real node", gotNode); - } else { - assertFalse("Not a PM scan - but got a null Node", gotNullNode); - assertTrue("Not a PM scan - without a real node", gotNode); - } - int deleted = gc.sweep(); - LOG.debug("Deleted " + deleted); - assertTrue("Should delete at least one item", deleted >= 0); - gc.close(); - } - } - - public String getNodeName(Node n) throws RepositoryException { - if (n == null) { - gotNullNode = true; - return String.valueOf(count++); - } else { - gotNode = true; - return n.getPath(); - } - } - - public void afterScanning(Node n) throws RepositoryException { - } - - public void beforeScanning(Node n) throws RepositoryException { - String s = getNodeName(n); - if (s != null) { - LOG.debug("scanning " + s); - } - } - - public void done() { - } - -} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCSubtreeMoveTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCSubtreeMoveTest.java (revision 1461410) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCSubtreeMoveTest.java (working copy) @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.core.data; - -import java.io.File; -import java.io.IOException; -import java.util.Iterator; -import java.util.Properties; - -import javax.jcr.Node; -import javax.jcr.RepositoryException; -import javax.jcr.Session; -import javax.jcr.SimpleCredentials; -import javax.jcr.ValueFactory; - -import junit.framework.TestCase; - -import org.apache.commons.io.FileUtils; -import org.apache.jackrabbit.api.JackrabbitRepository; -import org.apache.jackrabbit.api.JackrabbitRepositoryFactory; -import org.apache.jackrabbit.api.management.MarkEventListener; -import org.apache.jackrabbit.core.RepositoryFactoryImpl; -import org.apache.jackrabbit.core.SessionImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Test case for the scenario where the GC thread traverses the workspace and at - * some point, a subtree that the GC thread did not see yet is moved to a location - * that the thread has already traversed. The GC thread should not ignore binaries - * references by this subtree and eventually delete them. - */ -public class GCSubtreeMoveTest extends TestCase { - - private static final Logger logger = LoggerFactory.getLogger(GCSubtreeMoveTest.class); - - private String testDirectory; - private JackrabbitRepository repository; - private Session sessionGarbageCollector; - private Session sessionMover; - - public void setUp() throws IOException { - testDirectory = "target/" + getClass().getSimpleName() + "/" + getName(); - FileUtils.deleteDirectory(new File(testDirectory)); - } - - public void tearDown() throws IOException { - sessionGarbageCollector.logout(); - sessionMover.logout(); - repository.shutdown(); - - repository = null; - sessionGarbageCollector = null; - sessionMover = null; - - FileUtils.deleteDirectory(new File(testDirectory)); - testDirectory = null; - } - - public void test() { - setupRepository(); - - GarbageCollector garbageCollector = setupGarbageCollector(); - // To make sure even listener for NODE_ADDED is registered in GC. - garbageCollector.setPersistenceManagerScan(false); - - assertEquals(0, getBinaryCount(garbageCollector)); - setupNodes(); - assertEquals(1, getBinaryCount(garbageCollector)); - garbageCollector.getDataStore().clearInUse(); - - garbageCollector.setMarkEventListener(new MarkEventListener() { - - public void beforeScanning(Node node) throws RepositoryException { - String path = node.getPath(); - if (path.startsWith("/node")) { - log("Traversing: " + node.getPath()); - } - - if ("/node1".equals(node.getPath())) { - String from = "/node2/node3"; - String to = "/node0/node3"; - log("Moving " + from + " -> " + to); - sessionMover.move(from, to); - sessionMover.save(); - sleepForFile(); - } - } - }); - - try { - garbageCollector.getDataStore().clearInUse(); - garbageCollector.mark(); - garbageCollector.stopScan(); - sleepForFile(); - int numberOfDeleted = garbageCollector.sweep(); - log("Number of deleted: " + numberOfDeleted); - // Binary data should still be there. - assertEquals(1, getBinaryCount(garbageCollector)); - } catch (RepositoryException e) { - e.printStackTrace(); - failWithException(e); - } finally { - garbageCollector.close(); - } - } - - private void setupNodes() { - try { - Node rootNode = sessionMover.getRootNode(); - rootNode.addNode("node0"); - rootNode.addNode("node1"); - Node node2 = rootNode.addNode("node2"); - Node node3 = node2.addNode("node3"); - Node nodeWithBinary = node3.addNode("node-with-binary"); - ValueFactory vf = sessionGarbageCollector.getValueFactory(); - nodeWithBinary.setProperty("prop", vf.createBinary(new RandomInputStream(10, 1000))); - sessionMover.save(); - sleepForFile(); - } catch (RepositoryException e) { - failWithException(e); - } - } - - private void sleepForFile() { - // Make sure the file is old (access time resolution is 2 seconds) - try { - Thread.sleep(2200); - } catch (InterruptedException ignore) { - } - } - - private void setupRepository() { - JackrabbitRepositoryFactory repositoryFactory = new RepositoryFactoryImpl(); - createRepository(repositoryFactory); - login(); - } - - private void createRepository(JackrabbitRepositoryFactory repositoryFactory) { - Properties prop = new Properties(); - prop.setProperty("org.apache.jackrabbit.repository.home", testDirectory); - prop.setProperty("org.apache.jackrabbit.repository.conf", testDirectory + "/repository.xml"); - try { - repository = (JackrabbitRepository)repositoryFactory.getRepository(prop); - } catch (RepositoryException e) { - failWithException(e); - }; - } - - private void login() { - try { - sessionGarbageCollector = repository.login(new SimpleCredentials("admin", "admin".toCharArray())); - sessionMover = repository.login(new SimpleCredentials("admin", "admin".toCharArray())); - } catch (Exception e) { - failWithException(e); - } - } - - private GarbageCollector setupGarbageCollector() { - try { - return ((SessionImpl) sessionGarbageCollector).createDataStoreGarbageCollector(); - } catch (RepositoryException e) { - failWithException(e); - } - return null; - } - - private void failWithException(Exception e) { - fail("Not expected: " + e.getMessage()); - } - - private int getBinaryCount(GarbageCollector garbageCollector) { - int count = 0; - Iterator it; - try { - it = garbageCollector.getDataStore().getAllIdentifiers(); - while (it.hasNext()) { - it.next(); - count++; - } - } catch (DataStoreException e) { - failWithException(e); - } - log("Binary count: " + count); - return count; - } - - private void log(String message) { - logger.debug(message); - //System.out.println(message); - } -} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCThread.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCThread.java (revision 1461410) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/GCThread.java (working copy) @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.core.data; - -import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; -import org.apache.jackrabbit.api.management.MarkEventListener; -import org.apache.jackrabbit.core.SessionImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Iterator; - -import javax.jcr.Node; -import javax.jcr.RepositoryException; -import javax.jcr.Session; - -/** - * Helper class that runs data store garbage collection as a background thread. - */ -public class GCThread implements Runnable, MarkEventListener { - - /** logger instance */ - private static final Logger LOG = LoggerFactory.getLogger(GCThread.class); - - private boolean stop; - private Session session; - private Exception exception; - - public GCThread(Session session) { - this.session = session; - } - - public void run() { - - try { - GarbageCollector gc = ((SessionImpl) session) - .createDataStoreGarbageCollector(); - gc.setMarkEventListener(this); - while (!stop) { - LOG.debug("Scanning..."); - gc.mark(); - int count = listIdentifiers(gc); - LOG.debug("Stop; currently " + count + " identifiers"); - gc.stopScan(); - int numDeleted = gc.sweep(); - if (numDeleted > 0) { - LOG.debug("Deleted " + numDeleted + " identifiers"); - } - LOG.debug("Waiting..."); - Thread.sleep(10); - } - gc.close(); - } catch (Exception ex) { - LOG.error("Error scanning", ex); - exception = ex; - } - } - - public void setStop(boolean stop) { - this.stop = stop; - } - - public Exception getException() { - return exception; - } - - private int listIdentifiers(DataStoreGarbageCollector gc) throws DataStoreException { - DataStore ds = ((GarbageCollector) gc).getDataStore(); - Iterator it = ds.getAllIdentifiers(); - int count = 0; - while (it.hasNext()) { - DataIdentifier id = it.next(); - LOG.debug(" " + id); - count++; - } - return count; - } - - public void throwException() throws Exception { - if (exception != null) { - throw exception; - } - } - - public void beforeScanning(Node n) throws RepositoryException { - // nothing to do - } - -} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/TestAll.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/TestAll.java (revision 1461410) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/data/TestAll.java (working copy) @@ -35,15 +35,11 @@ public static Test suite() { TestSuite suite = new ConcurrentTestSuite("Data tests"); - suite.addTestSuite(ConcurrentGcTest.class); suite.addTestSuite(CopyValueTest.class); suite.addTestSuite(DataStoreAPITest.class); suite.addTestSuite(DataStoreTest.class); suite.addTestSuite(DBDataStoreTest.class); suite.addTestSuite(ExportImportTest.class); - suite.addTestSuite(GarbageCollectorTest.class); - suite.addTestSuite(GCConcurrentTest.class); - suite.addTestSuite(GCEventListenerTest.class); suite.addTestSuite(LazyFileInputStreamTest.class); suite.addTestSuite(NodeTypeTest.class); suite.addTestSuite(OpenFilesTest.class); @@ -51,7 +47,6 @@ suite.addTestSuite(TempFileInputStreamTest.class); suite.addTestSuite(TestTwoGetStreams.class); suite.addTestSuite(WriteWhileReadingTest.class); - suite.addTestSuite(GCSubtreeMoveTest.class); return suite; } Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/ConcurrentGcTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/ConcurrentGcTest.java (revision 0) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/ConcurrentGcTest.java (working copy) @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.core.gc; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; +import junit.framework.TestCase; +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.core.data.DataIdentifier; +import org.apache.jackrabbit.core.data.DataRecord; +import org.apache.jackrabbit.core.data.DataStore; +import org.apache.jackrabbit.core.data.DataStoreException; +import org.apache.jackrabbit.core.data.FileDataStore; +import org.apache.jackrabbit.core.data.db.DbDataStore; +import org.apache.jackrabbit.core.util.db.ConnectionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests concurrent garbage collection, see JCR-2026 + */ +public class ConcurrentGcTest extends TestCase { + + static final Logger LOG = LoggerFactory.getLogger(ConcurrentGcTest.class); + + private static final String TEST_DIR = "target/ConcurrentGcTest"; + + protected DataStore store; + private Thread gcLoopThread; + + protected Set ids = Collections.synchronizedSet(new HashSet()); + + protected volatile boolean gcLoopStop; + protected volatile Exception gcException; + + public void setUp() throws IOException { + deleteAll(); + } + + public void tearDown() throws IOException { + deleteAll(); + } + + private void deleteAll() throws IOException { + FileUtils.deleteDirectory(new File(TEST_DIR)); + } + + public void testDatabases() throws Exception { +// doTestDatabase( +// "org.h2.Driver", +// "jdbc:h2:" + TEST_DIR + "/db", +// "sa", "sa"); + + // not enabled by default + // doTestDatabase( + // "org.postgresql.Driver", + // "jdbc:postgresql:test", + // "sa", "sa"); + + // not enabled by default + // doTestDatabase( + // "com.mysql.jdbc.Driver", + // "jdbc:postgresql:test", + // "sa", "sa"); + + // fails with a deadlock + // doTestDatabase( + // "org.apache.derby.jdbc.EmbeddedDriver", + // "jdbc:derby:" + TEST_DIR + "/db;create=true", + // "sa", "sa"); + } + + private void doTestDatabase(String driver, String url, String user, String password) throws Exception { + ConnectionFactory pool = new ConnectionFactory(); + try { + DbDataStore store = new DbDataStore(); + store.setConnectionFactory(pool); + + ids.clear(); + + store.setDriver(driver); + store.setUrl(url); + store.setUser(user); + store.setPassword(password); + + store.init("target/test-db-datastore"); + store.setMinRecordLength(0); + doTest(store); + } finally { + pool.close(); + } + } + + public void testFile() throws Exception { + FileDataStore store = new FileDataStore(); + store.setPath(TEST_DIR + "/fs"); + store.init(TEST_DIR + "/fs"); + store.setMinRecordLength(0); + doTest(store); + } + + void doTest(DataStore store) throws Exception { + this.store = store; + + Random r = new Random(); + + concurrentGcLoopStart(); + + int len = 100; + if (getTestScale() > 1) { + len = 1000; + } + + for (int i = 0; i < len && gcException == null; i++) { + LOG.info("test " + i); + byte[] data = new byte[3]; + r.nextBytes(data); + DataRecord rec = store.addRecord(new ByteArrayInputStream(data)); + LOG.debug(" added " + rec.getIdentifier()); + if (r.nextBoolean()) { + LOG.debug(" added " + rec.getIdentifier() + " -> keep reference"); + ids.add(rec.getIdentifier()); + store.getRecord(rec.getIdentifier()); + } + if (r.nextInt(100) == 0) { + LOG.debug("clear i: " + i); + ids.clear(); + } + } + concurrentGcLoopStop(); + store.close(); + } + + private void concurrentGcLoopStart() { + gcLoopStop = false; + gcException = null; + + gcLoopThread = new Thread() { + public void run() { + try { + while (!gcLoopStop) { + if (ids.size() > 0) { + // store.clearInUse(); + long now = System.currentTimeMillis(); + LOG.debug("gc now: " + now); + store.updateModifiedDateOnAccess(now); + for (DataIdentifier id : new ArrayList(ids)) { + LOG.debug(" gc touch " + id); + store.getRecord(id); + } + int count = store.deleteAllOlderThan(now); + LOG.debug("gc now: " + now + " done, deleted: " + count); + } + } + } catch (DataStoreException e) { + gcException = e; + } + } + }; + gcLoopThread.start(); + } + + private void concurrentGcLoopStop() throws Exception { + gcLoopStop = true; + gcLoopThread.join(); + if (gcException != null) { + throw gcException; + } + } + + static int getTestScale() { + return Integer.parseInt(System.getProperty("jackrabbit.test.scale", "1")); + } + +} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GarbageCollectorTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GarbageCollectorTest.java (revision 0) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GarbageCollectorTest.java (working copy) @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.core.gc; + +import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; +import org.apache.jackrabbit.api.management.MarkEventListener; +import org.apache.jackrabbit.core.SessionImpl; +import org.apache.jackrabbit.core.data.DataIdentifier; +import org.apache.jackrabbit.core.data.DataStoreException; +import org.apache.jackrabbit.core.data.FileDataStore; +import org.apache.jackrabbit.core.data.GarbageCollector; +import org.apache.jackrabbit.core.data.RandomInputStream; +import org.apache.jackrabbit.core.data.ScanEventListener; +import org.apache.jackrabbit.test.AbstractJCRTest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import EDU.oswego.cs.dl.util.concurrent.SynchronousChannel; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; +import javax.jcr.Credentials; +import javax.jcr.Node; +import javax.jcr.NodeIterator; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.ValueFactory; + +/** + * Test cases for data store garbage collection. + */ +public class GarbageCollectorTest extends AbstractJCRTest implements ScanEventListener { + + /** logger instance */ + private static final Logger LOG = LoggerFactory.getLogger(GarbageCollectorTest.class); + + public void testCloseSessionWhileRunningGc() throws Exception { + final Session session = getHelper().getReadWriteSession(); + + final DataStoreGarbageCollector gc = ((SessionImpl) session).createDataStoreGarbageCollector(); + final Exception[] ex = new Exception[1]; + gc.setMarkEventListener(new MarkEventListener() { + boolean closed; + + public void beforeScanning(Node n) throws RepositoryException { + closeTest(); + } + + private void closeTest() throws RepositoryException { + if (closed) { + ex[0] = new Exception("Scanning after the session is closed"); + } + closed = true; + session.logout(); + } + + }); + try { + gc.mark(); + fail("Exception 'session has been closed' expected"); + } catch (RepositoryException e) { + LOG.debug("Expected exception caught: " + e.getMessage()); + } + if (ex[0] != null) { + throw ex[0]; + } + gc.close(); + } + + public void testConcurrentGC() throws Exception { + Node root = testRootNode; + Session session = root.getSession(); + + final SynchronousChannel sync = new SynchronousChannel(); + final Node node = root.addNode("slowBlob"); + final int blobLength = 1000; + final ValueFactory vf = session.getValueFactory(); + new Thread() { + public void run() { + try { + node.setProperty("slowBlob", vf.createBinary(new InputStream() { + int pos; + public int read() throws IOException { + pos++; + if (pos < blobLength) { + return pos % 80 == 0 ? '\n' : '.'; + } else if (pos == blobLength) { + try { + sync.put("x"); + // deleted + sync.take(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return 'x'; + } + return -1; + } + })); + node.getSession().save(); + sync.put("saved"); + } catch (Exception e) { + e.printStackTrace(); + } + } + }.start(); + assertEquals("x", sync.take()); + DataStoreGarbageCollector gc = ((SessionImpl) session).createDataStoreGarbageCollector(); + gc.mark(); + gc.sweep(); + sync.put("deleted"); + assertEquals("saved", sync.take()); + InputStream in = node.getProperty("slowBlob").getBinary().getStream(); + for (int pos = 1; pos < blobLength; pos++) { + int expected = pos % 80 == 0 ? '\n' : '.'; + assertEquals(expected, in.read()); + } + assertEquals('x', in.read()); + in.close(); + gc.close(); + } + + public void testGC() throws Exception { + Node root = testRootNode; + Session session = root.getSession(); + + deleteMyNodes(); + runGC(session, true); + + root.addNode("node1"); + Node node2 = root.addNode("node2"); + Node n = node2.addNode("nodeWithBlob"); + ValueFactory vf = session.getValueFactory(); + n.setProperty("test", vf.createBinary(new RandomInputStream(10, 1000))); + n = node2.addNode("nodeWithTemporaryBlob"); + n.setProperty("test", vf.createBinary(new RandomInputStream(11, 1000))); + session.save(); + + n.remove(); + session.save(); + + GarbageCollector gc = ((SessionImpl)session).createDataStoreGarbageCollector(); + + if (gc.getDataStore() instanceof FileDataStore) { + // make sure the file is old (access time resolution is 2 seconds) + Thread.sleep(2000); + } + + LOG.debug("scanning..."); + gc.mark(); + int count = listIdentifiers(gc); + LOG.debug("stop scanning; currently " + count + " identifiers"); + gc.stopScan(); + LOG.debug("deleting..."); + gc.getDataStore().clearInUse(); + assertTrue(gc.sweep() > 0); + int count2 = listIdentifiers(gc); + assertEquals(count - 1, count2); + + deleteMyNodes(); + + gc.close(); + } + + /** + * Test to validate that two GC cannot run simulatenously. one + * exits throwing exception + */ + public void testSimulatenousRunGC() throws Exception { + Node root = testRootNode; + Session session = root.getSession(); + + GCThread gct1 = new GCThread(session); + GCThread gct2 = new GCThread(session); + Thread gcThread1 = new Thread(gct1, "Datastore Garbage Collector 1"); + Thread gcThread2 = new Thread(gct2, "Datastore Garbage Collector 2"); + // run simulatensou gc + gcThread1.start(); + gcThread2.start(); + Thread.sleep(100); + + gct1.setStop(true); + gct2.setStop(true); + + // allow them to complete + gcThread1.join(); + gcThread2.join(); + + // only one should throw error + int count = (gct1.getException() == null ? 0 : 1) + (gct2.getException() == null ? 0 : 1); + assertEquals("only one gc should throw exception ", 1, count); + } + + private void runGC(Session session, boolean all) throws Exception { + GarbageCollector gc = ((SessionImpl)session).createDataStoreGarbageCollector(); + gc.setMarkEventListener(this); + if (gc.getDataStore() instanceof FileDataStore) { + // make sure the file is old (access time resolution is 2 seconds) + Thread.sleep(2000); + } + gc.mark(); + gc.stopScan(); + if (all) { + gc.getDataStore().clearInUse(); + } + gc.sweep(); + gc.close(); + } + + private int listIdentifiers(GarbageCollector gc) throws DataStoreException { + LOG.debug("identifiers:"); + int count = 0; + Iterator it = gc.getDataStore().getAllIdentifiers(); + while (it.hasNext()) { + DataIdentifier id = it.next(); + LOG.debug(" " + id); + count++; + } + return count; + } + + public void testTransientObjects() throws Exception { + + Node root = testRootNode; + Session session = root.getSession(); + + deleteMyNodes(); + + Credentials cred = getHelper().getSuperuserCredentials(); + Session s2 = getHelper().getRepository().login(cred); + root = s2.getRootNode(); + Node node2 = root.addNode("node3"); + Node n = node2.addNode("nodeWithBlob"); + ValueFactory vf = session.getValueFactory(); + n.setProperty("test", vf.createBinary(new RandomInputStream(10, 1000))); + + runGC(session, false); + + s2.save(); + + InputStream in = n.getProperty("test").getBinary().getStream(); + InputStream in2 = new RandomInputStream(10, 1000); + while (true) { + int a = in.read(); + int b = in2.read(); + assertEquals(a, b); + if (a < 0) { + break; + } + } + + deleteMyNodes(); + + s2.logout(); + } + + public void afterScanning(Node n) throws RepositoryException { + if (n != null && n.getPath().startsWith("/testroot/node")) { + String path = n.getPath(); + LOG.debug("scanned: " + path); + } + } + + private void list(Node n) throws RepositoryException { + if (!n.getName().startsWith("jcr:")) { + for (NodeIterator it = n.getNodes(); it.hasNext();) { + list(it.nextNode()); + } + } + } + + public void beforeScanning(Node n) throws RepositoryException { + if (n != null && n.getPath().equals("/testroot/node2")) { + Session session = n.getSession(); + list(session.getRootNode()); + session.move("/testroot/node2/nodeWithBlob", "/testroot/node1/nodeWithBlob"); + session.save(); + LOG.debug("moved /testroot/node2/nodeWithBlob to /testroot/node1"); + } + } + + private void deleteMyNodes() throws RepositoryException { + Node root = testRootNode; + while (root.hasNode("testroot")) { + root.getNode("testroot").remove(); + } + root.getSession().save(); + } + +} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCConcurrentTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCConcurrentTest.java (revision 0) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCConcurrentTest.java (working copy) @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.core.gc; + +import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; +import org.apache.jackrabbit.api.management.MarkEventListener; +import org.apache.jackrabbit.core.SessionImpl; +import org.apache.jackrabbit.test.AbstractJCRTest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Random; + +import javax.jcr.Node; +import javax.jcr.Property; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.ValueFactory; + +/** + * Test case for concurrent garbage collection + */ +public class GCConcurrentTest extends AbstractJCRTest { + + /** logger instance */ + private static final Logger LOG = LoggerFactory.getLogger(GCConcurrentTest.class); + + public void testConcurrentDelete() throws Exception { + Node root = testRootNode; + Session session = root.getSession(); + + final String testNodeName = "testConcurrentDelete"; + node(root, testNodeName); + session.save(); + DataStoreGarbageCollector gc = ((SessionImpl) session).createDataStoreGarbageCollector(); + gc.setPersistenceManagerScan(false); + gc.setMarkEventListener(new MarkEventListener() { + public void beforeScanning(Node n) throws RepositoryException { + if (n.getName().equals(testNodeName)) { + n.remove(); + n.getSession().save(); + } + } + + }); + gc.mark(); + gc.close(); + } + + public void testGC() throws Exception { + Node root = testRootNode; + Session session = root.getSession(); + + GCThread gc = new GCThread(session); + Thread gcThread = new Thread(gc, "Datastore Garbage Collector"); + + int len = 10 * getTestScale(); + boolean started = false; + for (int i = 0; i < len; i++) { + if (!started && i > 5 + len / 100) { + started = true; + gcThread.start(); + } + Node n = node(root, "test" + i); + ValueFactory vf = session.getValueFactory(); + n.setProperty("data", vf.createBinary(randomInputStream(i))); + session.save(); + LOG.debug("saved: " + i); + } + Thread.sleep(10); + for (int i = 0; i < len; i++) { + Node n = root.getNode("test" + i); + Property p = n.getProperty("data"); + InputStream in = p.getBinary().getStream(); + InputStream expected = randomInputStream(i); + checkStreams(expected, in); + n.remove(); + LOG.debug("removed: " + i); + session.save(); + } + Thread.sleep(10); + gc.setStop(true); + Thread.sleep(10); + gcThread.join(); + gc.throwException(); + } + + private void checkStreams(InputStream expected, InputStream in) throws IOException { + while (true) { + int e = expected.read(); + int i = in.read(); + if (e < 0 || i < 0) { + if (e >= 0 || i >= 0) { + fail("expected: " + e + " got: " + i); + } + break; + } else { + assertEquals(e, i); + } + } + expected.close(); + in.close(); + } + + static InputStream randomInputStream(long seed) { + byte[] data = new byte[4096]; + new Random(seed).nextBytes(data); + return new ByteArrayInputStream(data); + } + + static Node node(Node n, String x) throws RepositoryException { + return n.hasNode(x) ? n.getNode(x) : n.addNode(x); + } + + static int getTestScale() { + return Integer.parseInt(System.getProperty("jackrabbit.test.scale", "1")); + } + +} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCEventListenerTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCEventListenerTest.java (revision 0) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCEventListenerTest.java (working copy) @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.core.gc; + +import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; +import org.apache.jackrabbit.api.management.MarkEventListener; +import org.apache.jackrabbit.core.SessionImpl; +import org.apache.jackrabbit.core.data.DataStore; +import org.apache.jackrabbit.core.data.GarbageCollector; +import org.apache.jackrabbit.test.AbstractJCRTest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.util.Random; + +import javax.jcr.Node; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.ValueFactory; + +/** + * Test the DataStore garbage collector. + * This tests that the EventListener is called while scanning the repository. + * + * @author Thomas Mueller + */ +public class GCEventListenerTest extends AbstractJCRTest implements MarkEventListener { + + /** logger instance */ + private static final Logger LOG = LoggerFactory.getLogger(GCEventListenerTest.class); + + private static final String TEST_NODE_NAME = "testGCEventListener"; + + private boolean gotNullNode; + private boolean gotNode; + private int count; + + public void testEventListener() throws Exception { + doTestEventListener(true); + doTestEventListener(false); + } + + private void doTestEventListener(boolean allowPmScan) throws Exception { + Node root = testRootNode; + Session session = root.getSession(); + if (root.hasNode(TEST_NODE_NAME)) { + root.getNode(TEST_NODE_NAME).remove(); + session.save(); + } + Node test = root.addNode(TEST_NODE_NAME); + Random random = new Random(); + byte[] data = new byte[10000]; + for (int i = 0; i < 10; i++) { + Node n = test.addNode("x" + i); + random.nextBytes(data); + ValueFactory vf = session.getValueFactory(); + n.setProperty("data", vf.createBinary(new ByteArrayInputStream(data))); + session.save(); + if (i % 2 == 0) { + n.remove(); + session.save(); + } + } + session.save(); + SessionImpl si = (SessionImpl) session; + DataStoreGarbageCollector gc = si.createDataStoreGarbageCollector(); + DataStore ds = ((GarbageCollector) gc).getDataStore(); + if (ds != null) { + ds.clearInUse(); + boolean pmScan = gc.isPersistenceManagerScan(); + gc.setPersistenceManagerScan(allowPmScan); + gotNullNode = false; + gotNode = false; + gc.setMarkEventListener(this); + gc.mark(); + if (pmScan && allowPmScan) { + assertTrue("PM scan without null Node", gotNullNode); + assertFalse("PM scan, but got a real node", gotNode); + } else { + assertFalse("Not a PM scan - but got a null Node", gotNullNode); + assertTrue("Not a PM scan - without a real node", gotNode); + } + int deleted = gc.sweep(); + LOG.debug("Deleted " + deleted); + assertTrue("Should delete at least one item", deleted >= 0); + gc.close(); + } + } + + public String getNodeName(Node n) throws RepositoryException { + if (n == null) { + gotNullNode = true; + return String.valueOf(count++); + } else { + gotNode = true; + return n.getPath(); + } + } + + public void afterScanning(Node n) throws RepositoryException { + } + + public void beforeScanning(Node n) throws RepositoryException { + String s = getNodeName(n); + if (s != null) { + LOG.debug("scanning " + s); + } + } + + public void done() { + } + +} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCSubtreeMoveTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCSubtreeMoveTest.java (revision 0) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCSubtreeMoveTest.java (working copy) @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.core.gc; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.Properties; + +import javax.jcr.Node; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.SimpleCredentials; +import javax.jcr.ValueFactory; + +import junit.framework.TestCase; + +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.api.JackrabbitRepository; +import org.apache.jackrabbit.api.JackrabbitRepositoryFactory; +import org.apache.jackrabbit.api.management.MarkEventListener; +import org.apache.jackrabbit.core.RepositoryFactoryImpl; +import org.apache.jackrabbit.core.SessionImpl; +import org.apache.jackrabbit.core.data.DataIdentifier; +import org.apache.jackrabbit.core.data.DataStoreException; +import org.apache.jackrabbit.core.data.GarbageCollector; +import org.apache.jackrabbit.core.data.RandomInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test case for the scenario where the GC thread traverses the workspace and at + * some point, a subtree that the GC thread did not see yet is moved to a location + * that the thread has already traversed. The GC thread should not ignore binaries + * references by this subtree and eventually delete them. + */ +public class GCSubtreeMoveTest extends TestCase { + + private static final Logger logger = LoggerFactory.getLogger(GCSubtreeMoveTest.class); + + private String testDirectory; + private JackrabbitRepository repository; + private Session sessionGarbageCollector; + private Session sessionMover; + + public void setUp() throws IOException { + testDirectory = "target/" + getClass().getSimpleName() + "/" + getName(); + FileUtils.deleteDirectory(new File(testDirectory)); + } + + public void tearDown() throws IOException { + sessionGarbageCollector.logout(); + sessionMover.logout(); + repository.shutdown(); + + repository = null; + sessionGarbageCollector = null; + sessionMover = null; + + FileUtils.deleteDirectory(new File(testDirectory)); + testDirectory = null; + } + + public void test() { + setupRepository(); + + GarbageCollector garbageCollector = setupGarbageCollector(); + // To make sure even listener for NODE_ADDED is registered in GC. + garbageCollector.setPersistenceManagerScan(false); + + assertEquals(0, getBinaryCount(garbageCollector)); + setupNodes(); + assertEquals(1, getBinaryCount(garbageCollector)); + garbageCollector.getDataStore().clearInUse(); + + garbageCollector.setMarkEventListener(new MarkEventListener() { + + public void beforeScanning(Node node) throws RepositoryException { + String path = node.getPath(); + if (path.startsWith("/node")) { + log("Traversing: " + node.getPath()); + } + + if ("/node1".equals(node.getPath())) { + String from = "/node2/node3"; + String to = "/node0/node3"; + log("Moving " + from + " -> " + to); + sessionMover.move(from, to); + sessionMover.save(); + sleepForFile(); + } + } + }); + + try { + garbageCollector.getDataStore().clearInUse(); + garbageCollector.mark(); + garbageCollector.stopScan(); + sleepForFile(); + int numberOfDeleted = garbageCollector.sweep(); + log("Number of deleted: " + numberOfDeleted); + // Binary data should still be there. + assertEquals(1, getBinaryCount(garbageCollector)); + } catch (RepositoryException e) { + e.printStackTrace(); + failWithException(e); + } finally { + garbageCollector.close(); + } + } + + private void setupNodes() { + try { + Node rootNode = sessionMover.getRootNode(); + rootNode.addNode("node0"); + rootNode.addNode("node1"); + Node node2 = rootNode.addNode("node2"); + Node node3 = node2.addNode("node3"); + Node nodeWithBinary = node3.addNode("node-with-binary"); + ValueFactory vf = sessionGarbageCollector.getValueFactory(); + nodeWithBinary.setProperty("prop", vf.createBinary(new RandomInputStream(10, 1000))); + sessionMover.save(); + sleepForFile(); + } catch (RepositoryException e) { + failWithException(e); + } + } + + private void sleepForFile() { + // Make sure the file is old (access time resolution is 2 seconds) + try { + Thread.sleep(2200); + } catch (InterruptedException ignore) { + } + } + + private void setupRepository() { + JackrabbitRepositoryFactory repositoryFactory = new RepositoryFactoryImpl(); + createRepository(repositoryFactory); + login(); + } + + private void createRepository(JackrabbitRepositoryFactory repositoryFactory) { + Properties prop = new Properties(); + prop.setProperty("org.apache.jackrabbit.repository.home", testDirectory); + prop.setProperty("org.apache.jackrabbit.repository.conf", testDirectory + "/repository.xml"); + try { + repository = (JackrabbitRepository)repositoryFactory.getRepository(prop); + } catch (RepositoryException e) { + failWithException(e); + }; + } + + private void login() { + try { + sessionGarbageCollector = repository.login(new SimpleCredentials("admin", "admin".toCharArray())); + sessionMover = repository.login(new SimpleCredentials("admin", "admin".toCharArray())); + } catch (Exception e) { + failWithException(e); + } + } + + private GarbageCollector setupGarbageCollector() { + try { + return ((SessionImpl) sessionGarbageCollector).createDataStoreGarbageCollector(); + } catch (RepositoryException e) { + failWithException(e); + } + return null; + } + + private void failWithException(Exception e) { + fail("Not expected: " + e.getMessage()); + } + + private int getBinaryCount(GarbageCollector garbageCollector) { + int count = 0; + Iterator it; + try { + it = garbageCollector.getDataStore().getAllIdentifiers(); + while (it.hasNext()) { + it.next(); + count++; + } + } catch (DataStoreException e) { + failWithException(e); + } + log("Binary count: " + count); + return count; + } + + private void log(String message) { + logger.debug(message); + //System.out.println(message); + } +} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCThread.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCThread.java (revision 0) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/GCThread.java (working copy) @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.core.gc; + +import org.apache.jackrabbit.api.management.DataStoreGarbageCollector; +import org.apache.jackrabbit.api.management.MarkEventListener; +import org.apache.jackrabbit.core.SessionImpl; +import org.apache.jackrabbit.core.data.DataIdentifier; +import org.apache.jackrabbit.core.data.DataStore; +import org.apache.jackrabbit.core.data.DataStoreException; +import org.apache.jackrabbit.core.data.GarbageCollector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Iterator; + +import javax.jcr.Node; +import javax.jcr.RepositoryException; +import javax.jcr.Session; + +/** + * Helper class that runs data store garbage collection as a background thread. + */ +public class GCThread implements Runnable, MarkEventListener { + + /** logger instance */ + private static final Logger LOG = LoggerFactory.getLogger(GCThread.class); + + private boolean stop; + private Session session; + private Exception exception; + + public GCThread(Session session) { + this.session = session; + } + + public void run() { + + try { + GarbageCollector gc = ((SessionImpl) session) + .createDataStoreGarbageCollector(); + gc.setMarkEventListener(this); + while (!stop) { + LOG.debug("Scanning..."); + gc.mark(); + int count = listIdentifiers(gc); + LOG.debug("Stop; currently " + count + " identifiers"); + gc.stopScan(); + int numDeleted = gc.sweep(); + if (numDeleted > 0) { + LOG.debug("Deleted " + numDeleted + " identifiers"); + } + LOG.debug("Waiting..."); + Thread.sleep(10); + } + gc.close(); + } catch (Exception ex) { + LOG.error("Error scanning", ex); + exception = ex; + } + } + + public void setStop(boolean stop) { + this.stop = stop; + } + + public Exception getException() { + return exception; + } + + private int listIdentifiers(DataStoreGarbageCollector gc) throws DataStoreException { + DataStore ds = ((GarbageCollector) gc).getDataStore(); + Iterator it = ds.getAllIdentifiers(); + int count = 0; + while (it.hasNext()) { + DataIdentifier id = it.next(); + LOG.debug(" " + id); + count++; + } + return count; + } + + public void throwException() throws Exception { + if (exception != null) { + throw exception; + } + } + + public void beforeScanning(Node n) throws RepositoryException { + // nothing to do + } + +} Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/TestAll.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/TestAll.java (revision 0) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/gc/TestAll.java (working copy) @@ -0,0 +1,30 @@ +package org.apache.jackrabbit.core.gc; + + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Test suite to run GC tests. + * + */ + +public class TestAll extends TestCase{ + /** + * Returns a test suite that executes all tests inside this package. + * + * @return a test suite that executes all tests inside this package + */ + public static Test suite() { + /* GC test should run serially. Add them to a serial suite. */ + TestSuite suite = new TestSuite("GC tests"); + + suite.addTestSuite(ConcurrentGcTest.class); + suite.addTestSuite(GarbageCollectorTest.class); + suite.addTestSuite(GCConcurrentTest.class); + suite.addTestSuite(GCEventListenerTest.class); + suite.addTestSuite(GCSubtreeMoveTest.class); + return suite; + } +}