Index: src/test/java/org/apache/jackrabbit/oak/plugins/document/CountingDocumentStore.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/document/CountingDocumentStore.java (revision 1693074) +++ src/test/java/org/apache/jackrabbit/oak/plugins/document/CountingDocumentStore.java (working copy) @@ -219,4 +219,8 @@ return delegate.getMetadata(); } + @Override + public long determineServerTimeDifferenceMillis() { + return delegate.determineServerTimeDifferenceMillis(); + } } Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java (working copy) @@ -401,5 +401,11 @@ public Map getMetadata() { return metadata; } + + @Override + public long determineServerTimeDifferenceMillis() { + // the MemoryDocumentStore has no delays, thus return 0 + return 0; + } } \ No newline at end of file Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java (working copy) @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -122,7 +123,9 @@ private final DBCollection settings; private final DBCollection journal; - private final Cache nodesCache; + private final DB db; + + private final Cache nodesCache; private final CacheStats cacheStats; /** @@ -200,6 +203,7 @@ .put("version", version) .build(); + this.db = db; nodes = db.getCollection(Collection.NODES.toString()); clusterNodes = db.getCollection(Collection.CLUSTER_NODES.toString()); settings = db.getCollection(Collection.SETTINGS.toString()); @@ -1483,4 +1487,34 @@ parentLock.unlock(); } } + + @Override + public long determineServerTimeDifferenceMillis() { + // the assumption is that the network delay from this instance + // to the server, and from the server back to this instance + // are (more or less) equal. + // taking this assumption into account allows to remove + // the network delays from the picture: the difference + // between end and start time is exactly this network + // delay (plus some server time, but that's neglected). + // so if the clocks are in perfect sync and the above + // mentioned assumption holds, then the server time should + // be exactly at the midPoint between start and end. + // this should allow a more accurate picture of the diff. + final long start = System.currentTimeMillis(); + // assumption here: server returns UTC - ie the returned + // date object is correctly taking care of time zones. + final Date serverLocalTime = db.command("serverStatus").getDate("localTime"); + final long end = System.currentTimeMillis(); + + final long midPoint = (start/2) + (end/2); // to avoid risk of overflow: first half, then add + final long serverLocalTimeMillis = serverLocalTime.getTime(); + + // the difference should be + // * positive when local instance is ahead + // * and negative when the local instance is behind + final long diff = midPoint - serverLocalTimeMillis; + + return diff; + } } Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (working copy) @@ -446,6 +446,19 @@ observerTracker.start(context.getBundleContext()); DocumentStore ds = mk.getDocumentStore(); + + // OAK-2682: time difference detection applied at startup with a default max time diff of 2000 millis (2sec) + final long maxDiff = Long.parseLong(System.getProperty("oak.documentMK.maxServerTimeDiffMillis", "2000")); + try{ + final long timeDiff = ds.determineServerTimeDifferenceMillis(); + log.info("registerNodeStore: server time difference: {}ms (max allowed: {}ms)", timeDiff, maxDiff); + if (Math.abs(timeDiff)>Math.abs(maxDiff)) { + throw new IllegalStateException("Server clock seems off ("+timeDiff+"ms) by more than configured amount ("+maxDiff+"ms)"); + } + } catch(RuntimeException e) { // no checked exception + // in case of a RuntimeException, just log but continue + log.warn("registerNodeStore: got RuntimeException while trying to determine time difference to server: "+e, e); + } Dictionary props = new Hashtable(); props.put(Constants.SERVICE_PID, DocumentNodeStore.class.getName()); Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java (working copy) @@ -2224,4 +2224,9 @@ } return false; } + + @Override + public long determineServerTimeDifferenceMillis() { + throw new UnsupportedOperationException(); + } } Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java (working copy) @@ -308,4 +308,15 @@ * @return description of the underlying storage. */ Map getMetadata(); + + /** + * @return the estimated time difference in milliseconds between + * the local instance and the (typically common, shared) document server system. + * The value can be zero if the times are estimated to be equal, + * positive when the local instance is ahead of the remote server + * and negative when the local instance is behind the remote server. An invocation is not cached + * and typically requires a round-trip to the server (but that is not a requirement). + * @throws UnsupportedOperationException if this DocumentStore does not support this method + */ + long determineServerTimeDifferenceMillis(); } Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java (working copy) @@ -135,6 +135,11 @@ public synchronized CacheStats getCacheStats() { return store.getCacheStats(); } + + @Override + public synchronized long determineServerTimeDifferenceMillis() { + return store.determineServerTimeDifferenceMillis(); + } @Override public Map getMetadata() { Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java (working copy) @@ -333,6 +333,14 @@ public Map getMetadata() { return store.getMetadata(); } + + @Override + public long determineServerTimeDifferenceMillis() { + logMethod("determineServerTimeDifferenceMillis", "start"); + long result = store.determineServerTimeDifferenceMillis(); + logMethod("determineServerTimeDifferenceMillis", "end", result); + return result; + } private void logMethod(String methodName, Object... args) { StringBuilder buff = new StringBuilder("ds"); Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java (working copy) @@ -357,6 +357,18 @@ public Map getMetadata() { return base.getMetadata(); } + + @Override + public long determineServerTimeDifferenceMillis() { + try { + long start = now(); + long result = base.determineServerTimeDifferenceMillis(); + updateAndLogTimes("determineServerTimeDifferenceMillis", start, 0, 0); + return result; + } catch (Exception e) { + throw convert(e); + } + } private void logCommonCall(long start, String key) { int time = (int) (System.currentTimeMillis() - start); Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (working copy) @@ -2516,6 +2516,11 @@ } return sdf.format(r.getTimestamp()); } + + @Override + public long determineServerTimeDifferenceMillis() { + return store.determineServerTimeDifferenceMillis(); + } } static abstract class NodeStoreTask implements Runnable { Index: src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java (revision 1693074) +++ src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java (working copy) @@ -40,4 +40,15 @@ String[] getLastKnownRevisions(); String formatRevision(@Name("revision") String rev, @Name("UTC")boolean utc); + + /** + * @return the estimated time difference in milliseconds between + * the local instance and the (typically common, shared) document server system. + * The value can be zero if the times are estimated to be equal, + * positive when the local instance is ahead of the remote server + * and negative when the local instance is behind the remote server. An invocation is not cached + * and typically requires a round-trip to the server (but that is not a requirement). + * @throws UnsupportedOperationException if this DocumentStore does not support this method + */ + long determineServerTimeDifferenceMillis(); }