Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (revision 1669059) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (working copy) @@ -245,8 +245,10 @@ */ private volatile Revision headRevision; - private Thread backgroundThread; + private Thread backgroundReadThread; + private Thread backgroundUpdateThread; + /** * Background thread performing the clusterId lease renew. * Will be {@code null} if {@link #clusterNodeInfo} is {@code null}. @@ -439,15 +441,20 @@ dispatcher = new ChangeDispatcher(getRoot()); commitQueue = new CommitQueue(this, dispatcher); batchCommitQueue = new BatchCommitQueue(store, revisionComparator); - backgroundThread = new Thread( + backgroundReadThread = new Thread( + new BackgroundReadOperation(this, isDisposed), + "DocumentNodeStore background read thread"); + backgroundReadThread.setDaemon(true); + backgroundUpdateThread = new Thread( new BackgroundOperation(this, isDisposed), - "DocumentNodeStore background thread"); - backgroundThread.setDaemon(true); + "DocumentNodeStore background update thread"); + backgroundUpdateThread.setDaemon(true); checkLastRevRecovery(); // Renew the lease because it may have been stale renewClusterIdLease(); - backgroundThread.start(); + backgroundReadThread.start(); + backgroundUpdateThread.start(); if (clusterNodeInfo != null) { leaseUpdateThread = new Thread( @@ -478,14 +485,19 @@ isDisposed.notifyAll(); } try { - backgroundThread.join(); + backgroundReadThread.join(); } catch (InterruptedException e) { // ignore } + try { + backgroundUpdateThread.join(); + } catch (InterruptedException e) { + // ignore + } // do a final round of background operations after // the background thread stopped - internalRunBackgroundOperations(); + internalRunBackgroundUpdateOperations(); if (leaseUpdateThread != null) { try { @@ -1538,12 +1550,18 @@ //----------------------< background operations >--------------------------- + /** Used for testing only */ public void runBackgroundOperations() { + runBackgroundUpdateOperations(); + runBackgroundReadOperations(); + } + + private void runBackgroundUpdateOperations() { if (isDisposed.get()) { return; } try { - internalRunBackgroundOperations(); + internalRunBackgroundUpdateOperations(); } catch (RuntimeException e) { if (isDisposed.get()) { LOG.warn("Background operation failed: " + e.toString(), e); @@ -1553,7 +1571,7 @@ } } - private synchronized void internalRunBackgroundOperations() { + private synchronized void internalRunBackgroundUpdateOperations() { long start = clock.getTime(); long time = start; // clean orphaned branches and collisions @@ -1568,16 +1586,44 @@ // write back pending updates to _lastRev backgroundWrite(); long writeTime = clock.getTime() - time; - time = clock.getTime(); + String msg = "Background operations stats (clean:{}, split:{}, write:{})"; + if (clock.getTime() - start > TimeUnit.SECONDS.toMillis(10)) { + // log as info if it took more than 10 seconds + LOG.info(msg, cleanTime, splitTime, writeTime); + } else { + LOG.debug(msg, cleanTime, splitTime, writeTime); + } + } + + //----------------------< background read operations >---------------------- + + private void runBackgroundReadOperations() { + if (isDisposed.get()) { + return; + } + try { + internalRunBackgroundReadOperations(); + } catch (RuntimeException e) { + if (isDisposed.get()) { + LOG.warn("Background read operation failed: " + e.toString(), e); + return; + } + throw e; + } + } + + /** OAK-2624 : background read operations are split from background update ops */ + private synchronized void internalRunBackgroundReadOperations() { + long start = clock.getTime(); // pull in changes from other cluster nodes BackgroundReadStats readStats = backgroundRead(true); - long readTime = clock.getTime() - time; - String msg = "Background operations stats (clean:{}, split:{}, write:{}, read:{} {})"; + long readTime = clock.getTime() - start; + String msg = "Background read operations stats (read:{} {})"; if (clock.getTime() - start > TimeUnit.SECONDS.toMillis(10)) { // log as info if it took more than 10 seconds - LOG.info(msg, cleanTime, splitTime, writeTime, readTime, readStats); + LOG.info(msg, readTime, readStats); } else { - LOG.debug(msg, cleanTime, splitTime, writeTime, readTime, readStats); + LOG.debug(msg, readTime, readStats); } } @@ -2332,10 +2378,26 @@ @Override protected void execute(@Nonnull DocumentNodeStore nodeStore) { - nodeStore.runBackgroundOperations(); + nodeStore.runBackgroundUpdateOperations(); } } + /** + * Background read operations. + */ + static class BackgroundReadOperation extends NodeStoreTask { + + BackgroundReadOperation(DocumentNodeStore nodeStore, + AtomicBoolean isDisposed) { + super(nodeStore, isDisposed); + } + + @Override + protected void execute(@Nonnull DocumentNodeStore nodeStore) { + nodeStore.runBackgroundReadOperations(); + } + } + static class BackgroundLeaseUpdate extends NodeStoreTask { BackgroundLeaseUpdate(DocumentNodeStore nodeStore,