Index: oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java =================================================================== --- oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java (revision 1867491) +++ oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java (working copy) @@ -79,6 +79,7 @@ if (readOnly) { builder.setReadOnlyMode(); } + builder.setClusterInvisible(true); int cacheSize = docStoreOpts.getCacheSize(); if (cacheSize != 0) { Index: oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java =================================================================== --- oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java (revision 1867491) +++ oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java (working copy) @@ -20,8 +20,11 @@ package org.apache.jackrabbit.oak.run.cli; import java.io.IOException; +import java.util.List; import joptsimple.OptionParser; +import org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfoDocument; +import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder; import org.apache.jackrabbit.oak.plugins.document.MongoUtils; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; @@ -34,6 +37,7 @@ import static java.util.Collections.emptyMap; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -54,6 +58,7 @@ builder.setChildNode("foo"); store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); assertNotNull(fixture.getBlobStore()); + assertClusterInvisible(store); } } @@ -75,4 +80,11 @@ opts.parseAndConfigure(parser, new String[] {MongoUtils.URL}); return opts; } + + private void assertClusterInvisible(NodeStore store) { + List clusterInfos = + ClusterNodeInfoDocument.all(((DocumentNodeStore) store).getDocumentStore()); + assertNotNull(clusterInfos); + assertTrue(clusterInfos.get(0).isInvisible()); + } } Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java (revision 1867491) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java (working copy) @@ -166,6 +166,11 @@ private static final String READ_WRITE_MODE_KEY = "readWriteMode"; /** + * Key for invisible flag + */ + public static final String INVISIBLE = "invisible"; + + /** * The unique machine id (the MAC address if available). */ private static final String MACHINE_ID = getHardwareMachineId(); @@ -339,8 +344,14 @@ */ private LeaseFailureHandler leaseFailureHandler; + /** + * Flag to indicate this node is invisible to cluster view and thus recovery. + */ + private boolean invisible; + + private ClusterNodeInfo(int id, DocumentStore store, String machineId, - String instanceId, boolean newEntry) { + String instanceId, boolean newEntry, boolean invisible) { this.id = id; this.startTime = getCurrentTime(); this.leaseEndTime = this.startTime +leaseTime; @@ -349,6 +360,7 @@ this.machineId = machineId; this.instanceId = instanceId; this.newEntry = newEntry; + this.invisible = invisible; } void setLeaseCheckMode(@NotNull LeaseCheckMode mode) { @@ -371,6 +383,10 @@ return instanceId; } + boolean isInvisible() { + return invisible; + } + /** * Create a cluster node info instance to be utilized for read only access * to underlying store. @@ -379,7 +395,7 @@ * @return the cluster node info */ public static ClusterNodeInfo getReadOnlyInstance(DocumentStore store) { - return new ClusterNodeInfo(0, store, MACHINE_ID, WORKING_DIR, true) { + return new ClusterNodeInfo(0, store, MACHINE_ID, WORKING_DIR, true, false) { @Override public void dispose() { } @@ -418,10 +434,31 @@ * @return the cluster node info */ public static ClusterNodeInfo getInstance(DocumentStore store, + RecoveryHandler recoveryHandler, + String machineId, + String instanceId, + int configuredClusterId) { + + return getInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, false); + } + + /** + * Get or create a cluster node info instance for the store. + * + * @param store the document store (for the lease) + * @param recoveryHandler the recovery handler to call for a clusterId with + * an expired lease. + * @param machineId the machine id (null for MAC address) + * @param instanceId the instance id (null for current working directory) + * @param configuredClusterId the configured cluster id (or 0 for dynamic assignment) + * @return the cluster node info + */ + public static ClusterNodeInfo getInstance(DocumentStore store, RecoveryHandler recoveryHandler, String machineId, String instanceId, - int configuredClusterId) { + int configuredClusterId, + boolean invisible) { // defaults for machineId and instanceID if (machineId == null) { machineId = MACHINE_ID; @@ -434,7 +471,7 @@ for (int i = 0; i < retries; i++) { Map.Entry suggestedClusterNode = createInstance(store, recoveryHandler, machineId, - instanceId, configuredClusterId, i == 0); + instanceId, configuredClusterId, i == 0, invisible); ClusterNodeInfo clusterNode = suggestedClusterNode.getKey(); Long currentStartTime = suggestedClusterNode.getValue(); String key = String.valueOf(clusterNode.id); @@ -446,6 +483,7 @@ update.set(INFO_KEY, clusterNode.toString()); update.set(STATE, ACTIVE.name()); update.set(OAK_VERSION_KEY, OAK_VERSION); + update.set(INVISIBLE, invisible); ClusterNodeInfoDocument before = null; final boolean success; @@ -485,7 +523,8 @@ String machineId, String instanceId, int configuredClusterId, - boolean waitForLease) { + boolean waitForLease, + boolean invisible) { long now = getCurrentTime(); int maxId = 0; @@ -544,7 +583,7 @@ && iId.equals(instanceId)) { boolean worthRetrying = waitForLeaseExpiry(store, doc, leaseEnd, machineId, instanceId); if (worthRetrying) { - return createInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, false); + return createInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, false, invisible); } } @@ -579,7 +618,7 @@ // create a candidate. those with matching machine and instance id // are preferred, then the one with the lowest clusterId. - candidates.add(new ClusterNodeInfo(id, store, mId, iId, false)); + candidates.add(new ClusterNodeInfo(id, store, mId, iId, false, invisible)); startTimes.put(id, doc.getStartTime()); } @@ -596,13 +635,13 @@ clusterNodeId = maxId + 1; } // No usable existing entry found so create a new entry - candidates.add(new ClusterNodeInfo(clusterNodeId, store, machineId, instanceId, true)); + candidates.add(new ClusterNodeInfo(clusterNodeId, store, machineId, instanceId, true, invisible)); } // use the best candidate ClusterNodeInfo info = candidates.first(); // and replace with an info matching the current machine and instance id - info = new ClusterNodeInfo(info.id, store, machineId, instanceId, info.newEntry); + info = new ClusterNodeInfo(info.id, store, machineId, instanceId, info.newEntry, invisible); return new AbstractMap.SimpleImmutableEntry<>(info, startTimes.get(info.getId())); } @@ -609,6 +648,8 @@ private static void logClusterIdAcquired(ClusterNodeInfo clusterNode, ClusterNodeInfoDocument before) { String type = clusterNode.newEntry ? "new" : "existing"; + type = clusterNode.invisible ? (type + " (invisible)") : type; + String machineInfo = clusterNode.machineId; String instanceInfo = clusterNode.instanceId; if (before != null) { @@ -654,7 +695,7 @@ // check state of cluster node info ClusterNodeInfoDocument reread = store.find(Collection.CLUSTER_NODES, key); if (reread == null) { - LOG.info("Cluster node info " + key + ": gone; continueing."); + LOG.info("Cluster node info " + key + ": gone; continuing."); return true; } else { Long newLeaseEnd = (Long) reread.get(LEASE_END_KEY); @@ -1114,7 +1155,8 @@ "leaseCheckMode: " + leaseCheckMode.name() + ",\n" + "state: " + state + ",\n" + "oakVersion: " + OAK_VERSION + ",\n" + - "formatVersion: " + DocumentNodeStore.VERSION; + "formatVersion: " + DocumentNodeStore.VERSION + ",\n" + + "invisible: " + invisible; } /** Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java (revision 1867491) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java (working copy) @@ -157,4 +157,14 @@ public String getLastWrittenRootRev() { return (String) get(ClusterNodeInfo.LAST_WRITTEN_ROOT_REV_KEY); } + + /** + * Is the cluster node marked as invisible + * @return {@code true} if invisible; {@code false} + * otherwise. + */ + public boolean isInvisible() { + Boolean invisible = (Boolean) get(ClusterNodeInfo.INVISIBLE); + return invisible != null ? invisible : false; + } } Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBroadcastConfig.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBroadcastConfig.java (revision 1867491) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBroadcastConfig.java (working copy) @@ -41,7 +41,7 @@ public List> getClientInfo() { ArrayList> list = new ArrayList>(); for (ClusterNodeInfoDocument doc : ClusterNodeInfoDocument.all(documentNodeStore.getDocumentStore())) { - if (!doc.isActive()) { + if (!doc.isActive() || doc.isInvisible()) { continue; } Object broadcastId = doc.get(DynamicBroadcastConfig.ID); Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteService.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteService.java (revision 1867491) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteService.java (working copy) @@ -372,19 +372,21 @@ for (Iterator it = allClusterNodes.iterator(); it.hasNext();) { ClusterNodeInfoDocument clusterNode = it.next(); - allNodeIds.put(clusterNode.getClusterId(), clusterNode); - if (clusterNode.isBeingRecovered()) { - recoveringNodes.put(clusterNode.getClusterId(), clusterNode); - } else if (!clusterNode.isActive()) { - if (hasBacklog(clusterNode)) { - backlogNodes.put(clusterNode.getClusterId(), clusterNode); + if (!clusterNode.isInvisible()) { + allNodeIds.put(clusterNode.getClusterId(), clusterNode); + if (clusterNode.isBeingRecovered()) { + recoveringNodes.put(clusterNode.getClusterId(), clusterNode); + } else if (!clusterNode.isActive()) { + if (hasBacklog(clusterNode)) { + backlogNodes.put(clusterNode.getClusterId(), clusterNode); + } else { + inactiveNoBacklogNodes.put(clusterNode.getClusterId(), clusterNode); + } + } else if (clusterNode.getLeaseEndTime() < System.currentTimeMillis()) { + activeButTimedOutNodes.put(clusterNode.getClusterId(), clusterNode); } else { - inactiveNoBacklogNodes.put(clusterNode.getClusterId(), clusterNode); + activeNotTimedOutNodes.put(clusterNode.getClusterId(), clusterNode); } - } else if (clusterNode.getLeaseEndTime() < System.currentTimeMillis()) { - activeButTimedOutNodes.put(clusterNode.getClusterId(), clusterNode); - } else { - activeNotTimedOutNodes.put(clusterNode.getClusterId(), clusterNode); } } @@ -471,7 +473,8 @@ return null; } - private boolean hasBacklog(ClusterNodeInfoDocument clusterNode) { + /** package access only for testing **/ + boolean hasBacklog(ClusterNodeInfoDocument clusterNode) { if (logger.isTraceEnabled()) { logger.trace("hasBacklog: start. clusterNodeId: {}", clusterNode.getClusterId()); } @@ -653,7 +656,7 @@ * background-read has finished - as it could be waiting for a crashed * node's recovery to finish - which it can only do by checking the * lastKnownRevision of the crashed instance - and that check is best done - * after the background read is just finished (it could optinoally do that + * after the background read is just finished (it could optionally do that * just purely time based as well, but going via a listener is more timely, * that's why this approach has been chosen). */ Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (revision 1867491) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (working copy) @@ -562,7 +562,7 @@ } else { clusterNodeInfo = ClusterNodeInfo.getInstance(nonLeaseCheckingStore, new RecoveryHandlerImpl(nonLeaseCheckingStore, clock, lastRevSeeker), - null, null, cid); + null, null, cid, builder.isClusterInvisible()); checkRevisionAge(nonLeaseCheckingStore, clusterNodeInfo, clock); } this.clusterId = clusterNodeInfo.getId(); Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java (revision 1867491) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java (working copy) @@ -51,7 +51,6 @@ import org.apache.jackrabbit.oak.plugins.document.util.StringValue; import org.apache.jackrabbit.oak.spi.blob.AbstractBlobStore; import org.apache.jackrabbit.oak.spi.blob.BlobStore; -import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; import org.apache.jackrabbit.oak.spi.gc.GCMonitor; import org.apache.jackrabbit.oak.spi.gc.LoggingGCMonitor; @@ -151,6 +150,7 @@ private GCMonitor gcMonitor = new LoggingGCMonitor( LoggerFactory.getLogger(VersionGarbageCollector.class)); private Predicate nodeCachePredicate = Predicates.alwaysTrue(); + private boolean clusterInvisible; /** * @return a new {@link DocumentNodeStoreBuilder}. @@ -336,6 +336,18 @@ return thisBuilder(); } + /** + * Set the cluster as invisible to the discovery lite service. By default + * it is visible. + * + * @return this + * @see DocumentDiscoveryLiteService + */ + public T setClusterInvisible(boolean invisible) { + this.clusterInvisible = invisible; + return thisBuilder(); + } + public T setCacheSegmentCount(int cacheSegmentCount) { this.cacheSegmentCount = cacheSegmentCount; return thisBuilder(); @@ -350,6 +362,10 @@ return clusterId; } + public boolean isClusterInvisible() { + return clusterInvisible; + } + /** * Set the maximum delay to write the last revision to the root node. By * default 1000 (meaning 1 second) is used. Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java (revision 1867491) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java (working copy) @@ -85,7 +85,7 @@ */ class SimplifiedInstance { - private DocumentDiscoveryLiteService service; + DocumentDiscoveryLiteService service; DocumentNodeStore ns; private final Descriptors descriptors; private Map registeredServices; @@ -152,6 +152,10 @@ return Boolean.valueOf(finalStr); } + boolean isInvisible() { + return ns.getClusterInfo().isInvisible(); + } + boolean hasActiveIds(String clusterViewStr, int... expected) throws Exception { return hasIds(clusterViewStr, "active", expected); } @@ -464,9 +468,9 @@ class ViewExpectation implements Expectation { - private int[] activeIds; - private int[] deactivatingIds; - private int[] inactiveIds; + private int[] activeIds = new int[0]; + private int[] deactivatingIds = new int[0]; + private int[] inactiveIds = new int[0]; private final SimplifiedInstance discoveryLiteCombo; private boolean isFinal = true; @@ -526,7 +530,7 @@ if (!discoveryLiteCombo.hasInactiveIds(clusterViewStr, inactiveIds)) { return "inactiveIds dont match, expected: " + beautify(inactiveIds) + ", got clusterView: " + clusterViewStr; } - if (discoveryLiteCombo.isFinal() != isFinal) { + if (!discoveryLiteCombo.isInvisible() && discoveryLiteCombo.isFinal() != isFinal) { return "final flag does not match. expected: " + isFinal + ", but is: " + discoveryLiteCombo.isFinal(); } return null; @@ -579,6 +583,12 @@ // subsequent tests should get a DocumentDiscoveryLiteService setup from the // start DocumentNodeStore createNodeStore(String workingDir) throws SecurityException, Exception { + return createNodeStore(workingDir, false); + } + + // subsequent tests should get a DocumentDiscoveryLiteService setup from the + // start + DocumentNodeStore createNodeStore(String workingDir, boolean invisible) throws SecurityException, Exception { String prevWorkingDir = ClusterNodeInfo.WORKING_DIR; try { // ensure that we always get a fresh cluster[node]id @@ -587,7 +597,8 @@ // then create the DocumentNodeStore DocumentMK mk1 = createMK( 0 /* to make sure the clusterNodes collection is used **/, - 500 /* asyncDelay: background interval */); + 500, /* asyncDelay: background interval */ + invisible /* cluster node invisibility */); logger.info("createNodeStore: created DocumentNodeStore with cid=" + mk1.nodeStore.getClusterId() + ", workingDir=" + workingDir); @@ -599,12 +610,20 @@ } SimplifiedInstance createInstance() throws Exception { + return createInstance(false); + } + + SimplifiedInstance createInstance(boolean invisible) throws Exception { final String workingDir = UUID.randomUUID().toString(); - return createInstance(workingDir); + return createInstance(workingDir, invisible); } SimplifiedInstance createInstance(String workingDir) throws SecurityException, Exception { - DocumentNodeStore ns = createNodeStore(workingDir); + return createInstance(workingDir, false); + } + + SimplifiedInstance createInstance(String workingDir, boolean invisible) throws SecurityException, Exception { + DocumentNodeStore ns = createNodeStore(workingDir, invisible); return createInstance(ns, workingDir); } @@ -658,7 +677,9 @@ final List activeIds = new LinkedList(); for (Iterator it = instances.iterator(); it.hasNext();) { SimplifiedInstance anInstance = it.next(); - activeIds.add(anInstance.ns.getClusterId()); + if (!anInstance.isInvisible()) { + activeIds.add(anInstance.ns.getClusterId()); + } } logger.info("checkFiestaState: checking state. expected active: "+activeIds+", inactive: "+inactiveIds); for (Iterator it = instances.iterator(); it.hasNext();) { @@ -695,6 +716,11 @@ } DocumentMK createMK(int clusterId, int asyncDelay) { + return createMK(clusterId, asyncDelay, false); + } + + + DocumentMK createMK(int clusterId, int asyncDelay, boolean invisible) { if (MONGO_DB) { MongoConnection connection = connectionFactory.getConnection(); return register(new DocumentMK.Builder() @@ -708,13 +734,14 @@ if (bs == null) { bs = new MemoryBlobStore(); } - return createMK(clusterId, asyncDelay, ds, bs); + return createMK(clusterId, asyncDelay, ds, bs, invisible); } } - DocumentMK createMK(int clusterId, int asyncDelay, DocumentStore ds, BlobStore bs) { - return register(new DocumentMK.Builder().setDocumentStore(ds).setBlobStore(bs).setClusterId(clusterId).setLeaseCheckMode(LeaseCheckMode.DISABLED) - .setAsyncDelay(asyncDelay).open()); + DocumentMK createMK(int clusterId, int asyncDelay, DocumentStore ds, BlobStore bs, boolean invisible) { + return register(new DocumentMK.Builder().setDocumentStore(ds).setBlobStore(bs).setClusterId(clusterId).setClusterInvisible(invisible) + .setLeaseCheckMode(LeaseCheckMode.DISABLED) + .setAsyncDelay(asyncDelay).open()); } DocumentMK register(DocumentMK mk) { @@ -723,6 +750,19 @@ } /** + * Probability of invisible instance at 20% + * @param random + * @return + */ + boolean isInvisibleInstance(Random random) { + boolean invisible = false; + double invisibleProb = random.nextDouble(); + if (invisibleProb <= 0.2) { + invisible = true; + } + return invisible; + } + /** * This test creates a large number of documentnodestores which it starts, * runs, stops in a random fashion, always testing to make sure the * clusterView is correct @@ -758,7 +798,8 @@ logger.info("Case 0 - reactivated instance " + cid + ", workingDir=" + reactivatedWorkingDir); workingDir = reactivatedWorkingDir; logger.info("Case 0: creating instance"); - final SimplifiedInstance newInstance = createInstance(workingDir); + + final SimplifiedInstance newInstance = createInstance(workingDir, isInvisibleInstance(random)); newInstance.setLeastTimeout(5000, 1000); newInstance.startSimulatingWrites(500); logger.info("Case 0: created instance: " + newInstance.ns.getClusterId()); @@ -779,7 +820,7 @@ // creates a new instance if (instances.size() < MAX_NUM_INSTANCES) { logger.info("Case 1: creating instance"); - final SimplifiedInstance newInstance = createInstance(workingDir); + final SimplifiedInstance newInstance = createInstance(workingDir, isInvisibleInstance(random)); newInstance.setLeastTimeout(5000, 1000); newInstance.startSimulatingWrites(500); logger.info("Case 1: created instance: " + newInstance.ns.getClusterId()); @@ -803,7 +844,9 @@ final SimplifiedInstance instance = instances.remove(random.nextInt(instances.size())); assertNotNull(instance.workingDir); logger.info("Case 3: Shutdown instance: " + instance.ns.getClusterId()); - inactiveIds.put(instance.ns.getClusterId(), instance.workingDir); + if (!instance.isInvisible()) { + inactiveIds.put(instance.ns.getClusterId(), instance.workingDir); + } instance.shutdown(); } break; @@ -817,7 +860,9 @@ final SimplifiedInstance instance = instances.remove(random.nextInt(instances.size())); assertNotNull(instance.workingDir); logger.info("Case 4: Crashing instance: " + instance.ns.getClusterId()); - inactiveIds.put(instance.ns.getClusterId(), instance.workingDir); + if (!instance.isInvisible()) { + inactiveIds.put(instance.ns.getClusterId(), instance.workingDir); + } instance.addNode("/" + instance.ns.getClusterId() + "/stuffForRecovery/" + random.nextInt(10000)); instance.crash(); } Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java (revision 1867491) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java (working copy) @@ -144,7 +144,26 @@ } @Test - public void useAbandoned() throws InterruptedException { + public void useAbandonedStdToStd() throws InterruptedException { + useAbandoned(false, false); + } + + @Test + public void useAbandonedInvisibleToStd() throws InterruptedException { + useAbandoned(true, false); + } + + @Test + public void useAbandonedStdToInvisible() throws InterruptedException { + useAbandoned(false, true); + } + + @Test + public void useAbandonedInvisibleToInvisible() throws InterruptedException { + useAbandoned(true, true); + } + + public void useAbandoned(boolean firstInvisible, boolean secondInvisible) throws InterruptedException { Clock clock = new Clock.Virtual(); clock.waitUntil(System.currentTimeMillis()); ClusterNodeInfo.setClock(clock); @@ -155,6 +174,7 @@ clock(clock). setAsyncDelay(0). setLeaseCheckMode(LeaseCheckMode.DISABLED). + setClusterInvisible(firstInvisible). getNodeStore(); DocumentStore ds = ns1.getDocumentStore(); @@ -163,6 +183,7 @@ ClusterNodeInfoDocument cnid = ds.find(Collection.CLUSTER_NODES, "" + cid); assertNotNull(cnid); assertEquals(ClusterNodeState.ACTIVE.toString(), cnid.get(ClusterNodeInfo.STATE)); + assertEquals("Cluster should have been " + firstInvisible, firstInvisible, cnid.isInvisible()); ns1.dispose(); long waitFor = 2000; @@ -179,9 +200,16 @@ clock(clock). setAsyncDelay(0). setLeaseCheckMode(LeaseCheckMode.DISABLED). + setClusterInvisible(secondInvisible). getNodeStore(); assertEquals("should have re-used existing cluster id", cid, ns1.getClusterId()); + + cnid = ds.find(Collection.CLUSTER_NODES, "" + cid); + assertNotNull(cnid); + assertEquals(ClusterNodeState.ACTIVE.toString(), cnid.get(ClusterNodeInfo.STATE)); + assertEquals("Cluster should have been " + secondInvisible, secondInvisible, cnid.isInvisible()); + ns1.dispose(); } Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java (revision 1867491) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java (working copy) @@ -82,9 +82,9 @@ private ClusterNodeInfo newClusterNodeInfo(int id, String instanceId) { try { Constructor ctr = ClusterNodeInfo.class.getDeclaredConstructor( - int.class, DocumentStore.class, String.class, String.class, boolean.class); + int.class, DocumentStore.class, String.class, String.class, boolean.class, boolean.class); ctr.setAccessible(true); - return ctr.newInstance(id, store, MACHINE_ID, instanceId, true); + return ctr.newInstance(id, store, MACHINE_ID, instanceId, true, false); } catch (Exception e) { fail(e.getMessage()); } Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java (nonexistent) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java (working copy) @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.document; + +import java.util.List; + +import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore; +import org.junit.Test; + +import static org.apache.jackrabbit.oak.plugins.document.RecoveryHandler.NOOP; +import static org.hamcrest.Matchers.hasSize; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class ClusterNodeInfoDocumentTest { + + private DocumentStore store = new MemoryDocumentStore(); + + @Test + public void invisibleTrue() { + assertTrue(createInactive(true).isInvisible()); + } + + @Test + public void invisibleFalse() { + assertFalse(createInactive(false).isInvisible()); + } + + @Test + public void compatibility1_18() { + ClusterNodeInfoDocument doc = createInactive(false); + // remove invisible field introduced after 1.18 + UpdateOp op = new UpdateOp(String.valueOf(doc.getClusterId()), false); + op.remove(ClusterNodeInfo.INVISIBLE); + assertNotNull(store.findAndUpdate(Collection.CLUSTER_NODES, op)); + List docs = ClusterNodeInfoDocument.all(store); + assertThat(docs, hasSize(1)); + assertFalse(docs.get(0).isInvisible()); + } + + private ClusterNodeInfoDocument createInactive(boolean invisible) { + int clusterId = 1; + ClusterNodeInfo.getInstance(store, NOOP, "machineId", "instanceId", clusterId, invisible).dispose(); + return store.find(Collection.CLUSTER_NODES, String.valueOf(clusterId)); + } +} Property changes on: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java (revision 1867491) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java (working copy) @@ -28,11 +28,14 @@ import java.util.function.Function; import java.util.stream.Collectors; +import com.google.common.collect.Lists; import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore; import org.apache.jackrabbit.oak.stats.Clock; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -44,12 +47,23 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +@RunWith(Parameterized.class) public class ClusterNodeInfoTest { private Clock clock; private TestStore store; private FailureHandler handler = new FailureHandler(); + private boolean invisible; + public ClusterNodeInfoTest(boolean invisible) { + this.invisible = invisible; + } + + @Parameterized.Parameters(name="{index}: ({0})") + public static List fixtures() { + return Lists.newArrayList(false, true); + } + @Before public void before() throws Exception { clock = new Clock.Virtual(); @@ -551,7 +565,7 @@ private ClusterNodeInfo newClusterNodeInfo(int clusterId, String instanceId) { ClusterNodeInfo info = ClusterNodeInfo.getInstance(store, - new SimpleRecoveryHandler(), null, instanceId, clusterId); + new SimpleRecoveryHandler(), null, instanceId, clusterId, invisible); info.setLeaseFailureHandler(handler); return info; } Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java (nonexistent) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java (working copy) @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.document; + +import java.util.UUID; +import java.util.concurrent.Semaphore; + +import junitx.util.PrivateAccessor; +import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore; +import org.apache.jackrabbit.oak.stats.Clock; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import static org.junit.Assert.assertNotNull; +import static org.mockito.AdditionalAnswers.delegatesTo; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; + +public class DocumentDiscoveryLiteInvisibleServiceCrashTest + extends BaseDocumentDiscoveryLiteServiceTest { + + private static final int TEST_WAIT_TIMEOUT = 10000; + + private Clock clock; + private DocumentStore store; + private String wd1; + private DocumentNodeStore ns1; + private String wd2; + private DocumentNodeStore ns2; + + @Before + public void setup() throws Exception { + clock = new Clock.Virtual(); + clock.waitUntil(System.currentTimeMillis()); + ClusterNodeInfo.setClock(clock); + store = new MemoryDocumentStore(); + wd1 = UUID.randomUUID().toString(); + wd2 = UUID.randomUUID().toString(); + } + + @After + public void reset() { + ns1.dispose(); + ns2.dispose(); + ClusterNodeInfo.resetClockToDefault(); + } + + @Test + public void testTwoNodesWithCrashAndLongduringRecovery() throws Throwable { + doTestTwoNodesWithCrashAndLongduringDeactivation(false); + } + + @Test + public void testTwoNodesWithCrashAndLongduringRecoveryAndBacklog() throws Throwable { + doTestTwoNodesWithCrashAndLongduringDeactivation(true); + } + + private void doTestTwoNodesWithCrashAndLongduringDeactivation(boolean withBacklog) throws Throwable { + ns1 = newDocumentNodeStore(store, wd1); + SimplifiedInstance s1 = createInstance(ns1, wd1); + ViewExpectation e1 = new ViewExpectation(s1); + e1.setActiveIds(ns1.getClusterId()); + waitFor(e1, TEST_WAIT_TIMEOUT, "first should see itself active"); + ns1.runBackgroundOperations(); + + ns2 = newDocumentNodeStore(store, wd2, true); + SimplifiedInstance s2 = createInstance(ns2, wd2); + ViewExpectation e2 = new ViewExpectation(s2); + e2.setActiveIds(ns1.getClusterId()); + waitFor(e2, TEST_WAIT_TIMEOUT, "second should see only first active"); + ns2.runBackgroundOperations(); + + ns1.runBackgroundReadOperations(); + // now ns1 should also see both active + ViewExpectation e3 = new ViewExpectation(s1); + e3.setActiveIds(ns1.getClusterId()); + waitFor(e3, TEST_WAIT_TIMEOUT, "first should see only itself as active"); + + // before crashing s2, make sure that s1's lastRevRecovery thread + // doesn't run + s1.stopLastRevThread(); + if (withBacklog) { + // if we want to do backlog testing, then s2 should write + // something + // before it crashes, so here it comes: + s2.addNode("/foo/bar"); + s2.setProperty("/foo/bar", "prop", "value"); + } + + // then wait 2 sec + clock.waitUntil(clock.getTime() + 2000); + + s2.crash(); + + // then wait 2 sec + clock.waitUntil(clock.getTime() + 2000); + + // at this stage, while s2 has crashed, we have stopped s1's + // lastRevRecoveryThread, so we should still see both as active + logger.info(s1.getClusterViewStr()); + final ViewExpectation expectation1AfterCrashBeforeLastRevRecovery = new ViewExpectation(s1); + expectation1AfterCrashBeforeLastRevRecovery.setActiveIds(ns1.getClusterId()); + waitFor(expectation1AfterCrashBeforeLastRevRecovery, TEST_WAIT_TIMEOUT, "first should still see only itself as active"); + + // the next part is a bit tricky: we want to fine-control the + // lastRevRecoveryThread's acquire/release locking. + // the chosen way to do this is to make heavy use of mockito and two + // semaphores: + // when acquireRecoveryLock is called, that thread should wait for the + // waitBeforeLocking semaphore to be released + final MissingLastRevSeeker missingLastRevUtil = (MissingLastRevSeeker) PrivateAccessor + .getField(s1.ns.getLastRevRecoveryAgent(), "missingLastRevUtil"); + assertNotNull(missingLastRevUtil); + MissingLastRevSeeker mockedLongduringMissingLastRevUtil = mock(MissingLastRevSeeker.class, delegatesTo(missingLastRevUtil)); + final Semaphore waitBeforeLocking = new Semaphore(0); + doAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + logger.info("going to waitBeforeLocking"); + waitBeforeLocking.acquire(); + logger.info("done with waitBeforeLocking"); + return missingLastRevUtil.acquireRecoveryLock((Integer) invocation.getArguments()[0], + (Integer) invocation.getArguments()[1]); + } + }).when(mockedLongduringMissingLastRevUtil).acquireRecoveryLock(anyInt(), anyInt()); + PrivateAccessor.setField(s1.ns.getLastRevRecoveryAgent(), "missingLastRevUtil", mockedLongduringMissingLastRevUtil); + + // so let's start the lastRevThread again and wait for that + // waitBeforeLocking semaphore to be hit + s1.startLastRevThread(); + waitFor(new Expectation() { + + @Override + public String fulfilled() throws Exception { + if (!waitBeforeLocking.hasQueuedThreads()) { + return "no thread queued"; + } + return null; + } + + }, TEST_WAIT_TIMEOUT, "lastRevRecoveryThread should acquire a lock"); + + // at this stage the crashed s2 is still not in recovery mode, so let's + // check: + logger.info(s1.getClusterViewStr()); + final ViewExpectation expectation1AfterCrashBeforeLastRevRecoveryLocking = new ViewExpectation(s1); + expectation1AfterCrashBeforeLastRevRecoveryLocking.setActiveIds(ns1.getClusterId()); + waitFor(expectation1AfterCrashBeforeLastRevRecoveryLocking, TEST_WAIT_TIMEOUT, "first should still see itself as active"); + + // one thing, before we let the waitBeforeLocking go, setup the release + // semaphore/mock: + final Semaphore waitBeforeUnlocking = new Semaphore(0); + Mockito.doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) throws InterruptedException { + logger.info("Going to waitBeforeUnlocking"); + waitBeforeUnlocking.acquire(); + logger.info("Done with waitBeforeUnlocking"); + missingLastRevUtil.releaseRecoveryLock( + (Integer) invocation.getArguments()[0], + (Boolean) invocation.getArguments()[1]); + return null; + } + }).when(mockedLongduringMissingLastRevUtil).releaseRecoveryLock(anyInt(), anyBoolean()); + + // let go (or tschaedere loh) + waitBeforeLocking.release(); + + // then, right after we let the waitBeforeLocking semaphore go, we + // should see s2 in recovery mode + final ViewExpectation expectation1AfterCrashWhileLastRevRecoveryLocking = new ViewExpectation(s1); + expectation1AfterCrashWhileLastRevRecoveryLocking.setActiveIds(ns1.getClusterId()); + waitFor(expectation1AfterCrashWhileLastRevRecoveryLocking, TEST_WAIT_TIMEOUT, "first should still see itself as active"); + + // ok, meanwhile, the lastRevRecoveryAgent should have hit the ot + waitFor(new Expectation() { + + @Override + public String fulfilled() throws Exception { + if (!waitBeforeUnlocking.hasQueuedThreads()) { + return "no thread queued"; + } + return null; + } + + }, TEST_WAIT_TIMEOUT, "lastRevRecoveryThread should want to release a lock"); + + // so then, we should still see the same state + waitFor(expectation1AfterCrashWhileLastRevRecoveryLocking, TEST_WAIT_TIMEOUT, "first should still see itself as active"); + + logger.info("Waiting 2 sec"); + clock.waitUntil(clock.getTime() + 2000); + logger.info("Waiting done"); + + // first, lets check to see what the view looks like - should be + // unchanged: + waitFor(expectation1AfterCrashWhileLastRevRecoveryLocking, TEST_WAIT_TIMEOUT, "first should still see itself as active"); + + // let waitBeforeUnlocking go + logger.info("releasing waitBeforeUnlocking, state: " + s1.getClusterViewStr()); + waitBeforeUnlocking.release(); + logger.info("released waitBeforeUnlocking"); + + if (!withBacklog) { + final ViewExpectation expectationWithoutBacklog = new ViewExpectation(s1); + expectationWithoutBacklog.setActiveIds(ns1.getClusterId()); + waitFor(expectationWithoutBacklog, TEST_WAIT_TIMEOUT, "only first as active"); + waitFor(() -> { + if (!getLatestClusterInfo(ns2.getClusterId(), ns2).isActive() && + !getLatestClusterInfo(ns2.getClusterId(), ns2).isBeingRecovered()) { + return null; + } else { + return "Still not inactive"; + } + }, TEST_WAIT_TIMEOUT, "Second cluster should be inactive"); + } else { + // wait just 2 sec to see if the bgReadThread is really stopped + logger.info("sleeping 2 sec"); + clock.waitUntil(clock.getTime() + 2000); + logger.info("sleeping 2 sec done, state: " + s1.getClusterViewStr()); + + // when that's the case, check the view - it should now be in a + // special 'final=false' mode + final ViewExpectation expectationBeforeBgRead = new ViewExpectation(s1); + expectationBeforeBgRead.setActiveIds(ns1.getClusterId()); + waitFor(() -> { + ClusterNodeInfoDocument latestClusterInfo = getLatestClusterInfo(ns2.getClusterId(), ns2); + boolean hasBacklog = s1.service.hasBacklog(latestClusterInfo); + if (hasBacklog) { + return null; + } else { + return "No Backlog"; + } + }, TEST_WAIT_TIMEOUT, "Second cluster should have backlogs"); + waitFor(expectationBeforeBgRead, TEST_WAIT_TIMEOUT, "first should only see itself after shutdown"); + + // ook, now we explicitly do a background read to get out of the + // backlog situation + ns1.runBackgroundReadOperations(); + + final ViewExpectation expectationAfterBgRead = new ViewExpectation(s1); + expectationAfterBgRead.setActiveIds(ns1.getClusterId()); + waitFor(expectationAfterBgRead, TEST_WAIT_TIMEOUT, "we should see s1 as only active"); + waitFor(() -> { + ClusterNodeInfoDocument latestClusterInfo = getLatestClusterInfo(ns2.getClusterId(), ns2); + boolean hasBacklog = s1.service.hasBacklog(latestClusterInfo); + if (!hasBacklog) { + return null; + } else { + return "Still has Backlog"; + } + }, TEST_WAIT_TIMEOUT, "Second cluster should not have backlog any longer"); + } + } + + private static ClusterViewDocument read(DocumentNodeStore documentNodeStore) { + DocumentStore documentStore = documentNodeStore.getDocumentStore(); + Document doc = documentStore.find(Collection.SETTINGS, "clusterView", + -1 /* -1; avoid caching */); + if (doc == null) { + return null; + } else { + ClusterViewDocument clusterView = new ClusterViewDocument(doc); + if (clusterView.isValid()) { + return clusterView; + } else { + return null; + } + } + } + + private ClusterNodeInfoDocument getLatestClusterInfo(int id, DocumentNodeStore nodeStore) { + for (ClusterNodeInfoDocument doc : ClusterNodeInfoDocument.all(nodeStore.getDocumentStore())) { + int cId = doc.getClusterId(); + if (cId == id) { + return doc; + } + } + return null; + } + + private DocumentNodeStore newDocumentNodeStore(DocumentStore store, + String workingDir) { + return newDocumentNodeStore(store, workingDir, false); + } + + private DocumentNodeStore newDocumentNodeStore(DocumentStore store, + String workingDir, boolean invisible) { + String prevWorkingDir = ClusterNodeInfo.WORKING_DIR; + try { + // ensure that we always get a fresh cluster[node]id + ClusterNodeInfo.WORKING_DIR = workingDir; + + return new DocumentMK.Builder() + .clock(clock) + .setAsyncDelay(0) + .setDocumentStore(store) + .setLeaseCheckMode(LeaseCheckMode.DISABLED) + .setClusterInvisible(invisible) + .getNodeStore(); + } finally { + ClusterNodeInfo.WORKING_DIR = prevWorkingDir; + } + } + +} Property changes on: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceIT.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceIT.java (revision 1867491) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceIT.java (working copy) @@ -31,7 +31,7 @@ @Test public void testLargeStartStopFiesta() throws Throwable { logger.info("testLargeStartStopFiesta: start, seed="+SEED); - final int LOOP_CNT = 50; // with too many loops have also seen mongo + final int LOOP_CNT = 40; // with too many loops have also seen mongo // connections becoming starved thus test // failed doStartStopFiesta(LOOP_CNT); Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java (revision 1867491) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java (working copy) @@ -57,6 +57,14 @@ } @Test + public void testOneInvisibleNode() throws Exception { + final SimplifiedInstance s1 = createInstance(true); + final ViewExpectation expectation = new ViewExpectation(s1); + expectation.setActiveIds(new int[0]); + waitFor(expectation, 2000, "no one is active"); + } + + @Test public void testTwoNodesWithCleanShutdown() throws Exception { final SimplifiedInstance s1 = createInstance(); final SimplifiedInstance s2 = createInstance(); @@ -75,6 +83,23 @@ } @Test + public void testTwoNodesWithInvisibleCleanShutdown() throws Exception { + final SimplifiedInstance s1 = createInstance(true); + final SimplifiedInstance s2 = createInstance(); + final ViewExpectation expectation1 = new ViewExpectation(s1); + final ViewExpectation expectation2 = new ViewExpectation(s2); + expectation1.setActiveIds(s2.ns.getClusterId()); + expectation2.setActiveIds(s2.ns.getClusterId()); + waitFor(expectation1, 2000, "Only second is active"); + waitFor(expectation2, 2000, "Second should not see first as active"); + + s1.shutdown(); + final ViewExpectation expectation1AfterShutdown = new ViewExpectation(s2); + expectation1AfterShutdown.setActiveIds(s2.ns.getClusterId()); + waitFor(expectation1AfterShutdown, 2000, "no one is active after shutdown"); + } + + @Test public void testTwoNodesWithCrash() throws Throwable { final SimplifiedInstance s1 = createInstance(); final SimplifiedInstance s2 = createInstance(); @@ -93,6 +118,24 @@ waitFor(expectation1AfterShutdown, 4000, "first should only see itself after shutdown"); } + @Test + public void testTwoNodesInvisibleWithCrash() throws Throwable { + final SimplifiedInstance s1 = createInstance(true); + final SimplifiedInstance s2 = createInstance(); + final ViewExpectation expectation1 = new ViewExpectation(s1); + final ViewExpectation expectation2 = new ViewExpectation(s2); + expectation1.setActiveIds(s2.ns.getClusterId()); + expectation2.setActiveIds(s2.ns.getClusterId()); + waitFor(expectation1, 2000, "first should see only second as active"); + waitFor(expectation2, 2000, "second should not see first as active"); + + s1.crash(); + + final ViewExpectation expectation1AfterShutdown = new ViewExpectation(s1); + expectation1AfterShutdown.setActiveIds(s2.ns.getClusterId()); + waitFor(expectation1AfterShutdown, 4000, "first should only see itself after shutdown"); + } + /** * This test creates a large number of documentnodestores which it starts, * runs, stops in a random fashion, always testing to make sure the