From ac6d75dc55eab0ce909c972a1124bf5fb42a5e3d Mon Sep 17 00:00:00 2001 From: Denis Magda Date: Mon, 3 Aug 2015 12:44:45 +0300 Subject: [PATCH 1/8] ignite-1189: clearing grid information on stop even when grid's state is not STARTED --- .../java/org/apache/ignite/internal/IgnitionEx.java | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index 5cbe377..d355085 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -261,8 +261,11 @@ public class IgnitionEx { public static boolean stop(@Nullable String name, boolean cancel) { IgniteNamedInstance grid = name != null ? grids.get(name) : dfltGrid; - if (grid != null && grid.state() == STARTED) { - grid.stop(cancel); + if (grid != null) { + IgniteState state = grid.state(); + + if (state == STARTED) + grid.stop(cancel); boolean fireEvt; @@ -277,10 +280,18 @@ public class IgnitionEx { } } - if (fireEvt) - notifyStateChange(grid.getName(), grid.state()); + if (state == STARTED) { + if (fireEvt) + notifyStateChange(grid.getName(), grid.state()); - return true; + return true; + } + else { + U.warn(null, "Ignoring stopping grid instance (has not been in STARTED state): [grid=" + name + + ", state=" + state + ']'); + + return false; + } } // We don't have log at this point... -- 1.9.5.msysgit.0 From 6b237e119caad68474cc785c0373da8ce31011d8 Mon Sep 17 00:00:00 2001 From: Denis Magda Date: Mon, 3 Aug 2015 14:45:40 +0300 Subject: [PATCH 2/8] ignite-1189: trying to reproduce failures --- .../java/org/apache/ignite/internal/IgnitionEx.java | 21 +++++---------------- .../IgniteCacheAtomicNodeRestartTest.java | 4 ++++ 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index d355085..5cbe377 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -261,11 +261,8 @@ public class IgnitionEx { public static boolean stop(@Nullable String name, boolean cancel) { IgniteNamedInstance grid = name != null ? grids.get(name) : dfltGrid; - if (grid != null) { - IgniteState state = grid.state(); - - if (state == STARTED) - grid.stop(cancel); + if (grid != null && grid.state() == STARTED) { + grid.stop(cancel); boolean fireEvt; @@ -280,18 +277,10 @@ public class IgnitionEx { } } - if (state == STARTED) { - if (fireEvt) - notifyStateChange(grid.getName(), grid.state()); - - return true; - } - else { - U.warn(null, "Ignoring stopping grid instance (has not been in STARTED state): [grid=" + name + - ", state=" + state + ']'); + if (fireEvt) + notifyStateChange(grid.getName(), grid.state()); - return false; - } + return true; } // We don't have log at this point... diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java index 1c4e616..fa8898f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java @@ -30,4 +30,8 @@ public class IgniteCacheAtomicNodeRestartTest extends GridCachePartitionedNodeRe @Override protected CacheAtomicityMode atomicityMode() { return ATOMIC; } + + @Override public void testRestartWithPutTenNodesTwoBackups() throws Throwable { + + } } -- 1.9.5.msysgit.0 From efa7e99bbad76ce35a55e6ad9faa9aac8e57b5f4 Mon Sep 17 00:00:00 2001 From: Denis Magda Date: Tue, 4 Aug 2015 09:12:12 +0300 Subject: [PATCH 3/8] ignite-1189: reproducing deadlock --- .../distributed/dht/atomic/GridDhtAtomicCache.java | 36 +++++++++++++++++- .../IgniteCacheAtomicNodeRestartTest.java | 43 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 0a21979..18911fd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -85,6 +85,8 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { /** */ private GridNearAtomicCache near; + private ThreadLocal> lockedEntries = new ThreadLocal<>(); + /** * Empty constructor required by {@link Externalizable}. */ @@ -990,6 +992,20 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { final GridNearAtomicUpdateRequest req, final CI2 completionCb ) { + boolean printKeys = false; + + if (lockedEntries.get() != null) { + for (GridDhtCacheEntry entry : lockedEntries.get()) + U.error(log, "Locked entry [entry=" + entry + ']'); + + printKeys = true; + } + + if (printKeys) { + for (KeyCacheObject obj : req.keys()) + U.error(log, "Key requested: " + obj); + } + IgniteInternalFuture forceFut = preldr.request(req.keys(), req.topologyVersion()); if (forceFut.isDone()) @@ -1032,10 +1048,26 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { IgniteCacheExpiryPolicy expiry = null; try { + boolean printKeys = false; + + if (lockedEntries.get() != null) { + for (GridDhtCacheEntry entry : lockedEntries.get()) + U.error(log, "Locked entry (2) [entry=" + entry + ']'); + + printKeys = true; + } + + if (printKeys) { + for (KeyCacheObject obj : keys) + U.error(log, "Key requested: " + obj); + } + // If batch store update is enabled, we need to lock all entries. // First, need to acquire locks on cache entries, then check filter. List locked = lockEntries(keys, req.topologyVersion()); + lockedEntries.set(locked); + Collection> deleted = null; try { @@ -1153,8 +1185,10 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { e.printStackTrace(); } finally { - if (locked != null) + if (locked != null) { + lockedEntries.set(null); unlockEntries(locked, req.topologyVersion()); + } // Enqueue if necessary after locks release. if (deleted != null) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java index fa8898f..70e6c4c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java @@ -31,7 +31,50 @@ public class IgniteCacheAtomicNodeRestartTest extends GridCachePartitionedNodeRe return ATOMIC; } + /** {@inheritDoc} */ @Override public void testRestartWithPutTenNodesTwoBackups() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + + @Override public void testRestart() throws Exception { + } + + @Override public void testRestartWithPutTwoNodesNoBackups() throws Throwable { + } + + @Override public void testRestartWithPutTwoNodesOneBackup() throws Throwable { + } + + @Override public void testRestartWithPutFourNodesNoBackups() throws Throwable { + } + + @Override public void testRestartWithPutFourNodesOneBackups() throws Throwable { + } + + @Override public void testRestartWithPutSixNodesTwoBackups() throws Throwable { + } + + @Override public void testRestartWithPutEightNodesTwoBackups() throws Throwable { + } + + @Override public void testRestartWithTxEightNodesTwoBackups() throws Throwable { + } + + @Override public void testRestartWithTxFourNodesNoBackups() throws Throwable { + } + + @Override public void testRestartWithTxFourNodesOneBackups() throws Throwable { + } + + @Override public void testRestartWithTxSixNodesTwoBackups() throws Throwable { + } + + @Override public void testRestartWithTxTenNodesTwoBackups() throws Throwable { + } + + @Override public void testRestartWithTxTwoNodesNoBackups() throws Throwable { + } + @Override public void testRestartWithTxTwoNodesOneBackup() throws Throwable { } } -- 1.9.5.msysgit.0 From 3ce3c8b71064ff4d8abd78cd95f4678cb7a74811 Mon Sep 17 00:00:00 2001 From: Denis Magda Date: Tue, 4 Aug 2015 10:33:57 +0300 Subject: [PATCH 4/8] ignite-1189: 2 reproducing deadlock --- .../distributed/dht/atomic/GridDhtAtomicCache.java | 40 +++------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 18911fd..d6163c2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -85,8 +85,6 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { /** */ private GridNearAtomicCache near; - private ThreadLocal> lockedEntries = new ThreadLocal<>(); - /** * Empty constructor required by {@link Externalizable}. */ @@ -992,20 +990,6 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { final GridNearAtomicUpdateRequest req, final CI2 completionCb ) { - boolean printKeys = false; - - if (lockedEntries.get() != null) { - for (GridDhtCacheEntry entry : lockedEntries.get()) - U.error(log, "Locked entry [entry=" + entry + ']'); - - printKeys = true; - } - - if (printKeys) { - for (KeyCacheObject obj : req.keys()) - U.error(log, "Key requested: " + obj); - } - IgniteInternalFuture forceFut = preldr.request(req.keys(), req.topologyVersion()); if (forceFut.isDone()) @@ -1048,26 +1032,10 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { IgniteCacheExpiryPolicy expiry = null; try { - boolean printKeys = false; - - if (lockedEntries.get() != null) { - for (GridDhtCacheEntry entry : lockedEntries.get()) - U.error(log, "Locked entry (2) [entry=" + entry + ']'); - - printKeys = true; - } - - if (printKeys) { - for (KeyCacheObject obj : keys) - U.error(log, "Key requested: " + obj); - } - // If batch store update is enabled, we need to lock all entries. // First, need to acquire locks on cache entries, then check filter. List locked = lockEntries(keys, req.topologyVersion()); - lockedEntries.set(locked); - Collection> deleted = null; try { @@ -1184,11 +1152,13 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { e.printStackTrace(); } + catch (Exception e) { + if (X.hasCause(e, InterruptedException.class)) + U.error(log, "FUCK Interrupted", e); + } finally { - if (locked != null) { - lockedEntries.set(null); + if (locked != null) unlockEntries(locked, req.topologyVersion()); - } // Enqueue if necessary after locks release. if (deleted != null) { -- 1.9.5.msysgit.0 From 28c9977bb8344f2d73c65d45656b673a9a4c634b Mon Sep 17 00:00:00 2001 From: Denis Magda Date: Tue, 4 Aug 2015 11:41:48 +0300 Subject: [PATCH 5/8] ignite-1189: 3 reproducing deadlock --- .../processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index d6163c2..4d73fb2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -1153,8 +1153,9 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { e.printStackTrace(); } catch (Exception e) { - if (X.hasCause(e, InterruptedException.class)) - U.error(log, "FUCK Interrupted", e); + U.error(log, "FUCK ERROR", e); + + throw e; } finally { if (locked != null) -- 1.9.5.msysgit.0 From 132562b431bc3250abe56ec48725c5ebe6083964 Mon Sep 17 00:00:00 2001 From: Denis Magda Date: Tue, 4 Aug 2015 12:26:00 +0300 Subject: [PATCH 6/8] ignite-1189: 4 reproducing deadlock --- .../distributed/dht/atomic/GridDhtAtomicCache.java | 8 ++-- .../IgniteCacheAtomicNodeRestartTest.java | 45 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 4d73fb2..cd6e28d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -1152,11 +1152,6 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { e.printStackTrace(); } - catch (Exception e) { - U.error(log, "FUCK ERROR", e); - - throw e; - } finally { if (locked != null) unlockEntries(locked, req.topologyVersion()); @@ -1179,6 +1174,9 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { remap = true; } + catch (Exception e) { + U.error(log, "Unexpected exception during cache update", e); + } if (remap) { assert dhtFut == null; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java index 70e6c4c..caee4d0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java @@ -36,7 +36,52 @@ public class IgniteCacheAtomicNodeRestartTest extends GridCachePartitionedNodeRe super.testRestartWithPutTenNodesTwoBackups(); } + public void testRestartWithPutTenNodesTwoBackups2() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups3() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups4() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups5() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups6() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups7() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups8() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups9() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups10() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups11() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups12() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups13() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + public void testRestartWithPutTenNodesTwoBackups14() throws Throwable { + super.testRestartWithPutTenNodesTwoBackups(); + } + + @Override protected long getTestTimeout() { + return Long.MAX_VALUE; + } + @Override public void testRestart() throws Exception { + } @Override public void testRestartWithPutTwoNodesNoBackups() throws Throwable { -- 1.9.5.msysgit.0 From 4d528becc1cd3db9d4d2d6db2053895043aa3918 Mon Sep 17 00:00:00 2001 From: Denis Magda Date: Tue, 4 Aug 2015 13:04:35 +0300 Subject: [PATCH 7/8] ignite-1189: fixing deadlock --- .../distributed/dht/atomic/GridDhtAtomicCache.java | 29 ++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index cd6e28d..470efdd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -1176,6 +1176,12 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { } catch (Exception e) { U.error(log, "Unexpected exception during cache update", e); + + res.addFailedKeys(keys, e); + + completionCb.apply(req, res); + + return; } if (remap) { @@ -2167,19 +2173,22 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { // Enqueue entries while holding locks. Collection skip = null; - for (GridCacheMapEntry entry : locked) { - if (entry != null && entry.deleted()) { - if (skip == null) - skip = new HashSet<>(locked.size(), 1.0f); + try { + for (GridCacheMapEntry entry : locked) { + if (entry != null && entry.deleted()) { + if (skip == null) + skip = new HashSet<>(locked.size(), 1.0f); - skip.add(entry.key()); + skip.add(entry.key()); + } } } - - // Release locks. - for (GridCacheMapEntry entry : locked) { - if (entry != null) - UNSAFE.monitorExit(entry); + finally { + // Release locks. + for (GridCacheMapEntry entry : locked) { + if (entry != null) + UNSAFE.monitorExit(entry); + } } // Try evict partitions. -- 1.9.5.msysgit.0 From 67706063d2e8d8cc3ed8d55cdfffccc0a21005c6 Mon Sep 17 00:00:00 2001 From: Denis Magda Date: Tue, 4 Aug 2015 13:34:19 +0300 Subject: [PATCH 8/8] ignite-1189: eventually fixed the deadlock --- .../distributed/dht/atomic/GridDhtAtomicCache.java | 6 +- .../IgniteCacheAtomicNodeRestartTest.java | 92 ---------------------- 2 files changed, 5 insertions(+), 93 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 470efdd..14b4680 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -1175,6 +1175,8 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { remap = true; } catch (Exception e) { + // At least RuntimeException can be thrown by the code above when GridCacheContext is cleaned and there is + // an attempt to use cleaned resources. U.error(log, "Unexpected exception during cache update", e); res.addFailedKeys(keys, e); @@ -2184,7 +2186,9 @@ public class GridDhtAtomicCache extends GridDhtCacheAdapter { } } finally { - // Release locks. + // At least RuntimeException can be thrown by the code above when GridCacheContext is cleaned and there is + // an attempt to use cleaned resources. + // That's why releasing locks in the finally block.. for (GridCacheMapEntry entry : locked) { if (entry != null) UNSAFE.monitorExit(entry); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java index caee4d0..1c4e616 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicNodeRestartTest.java @@ -30,96 +30,4 @@ public class IgniteCacheAtomicNodeRestartTest extends GridCachePartitionedNodeRe @Override protected CacheAtomicityMode atomicityMode() { return ATOMIC; } - - /** {@inheritDoc} */ - @Override public void testRestartWithPutTenNodesTwoBackups() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - - public void testRestartWithPutTenNodesTwoBackups2() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups3() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups4() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups5() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups6() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups7() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups8() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups9() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups10() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups11() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups12() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups13() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - public void testRestartWithPutTenNodesTwoBackups14() throws Throwable { - super.testRestartWithPutTenNodesTwoBackups(); - } - - @Override protected long getTestTimeout() { - return Long.MAX_VALUE; - } - - @Override public void testRestart() throws Exception { - - } - - @Override public void testRestartWithPutTwoNodesNoBackups() throws Throwable { - } - - @Override public void testRestartWithPutTwoNodesOneBackup() throws Throwable { - } - - @Override public void testRestartWithPutFourNodesNoBackups() throws Throwable { - } - - @Override public void testRestartWithPutFourNodesOneBackups() throws Throwable { - } - - @Override public void testRestartWithPutSixNodesTwoBackups() throws Throwable { - } - - @Override public void testRestartWithPutEightNodesTwoBackups() throws Throwable { - } - - @Override public void testRestartWithTxEightNodesTwoBackups() throws Throwable { - } - - @Override public void testRestartWithTxFourNodesNoBackups() throws Throwable { - } - - @Override public void testRestartWithTxFourNodesOneBackups() throws Throwable { - } - - @Override public void testRestartWithTxSixNodesTwoBackups() throws Throwable { - } - - @Override public void testRestartWithTxTenNodesTwoBackups() throws Throwable { - } - - @Override public void testRestartWithTxTwoNodesNoBackups() throws Throwable { - } - - @Override public void testRestartWithTxTwoNodesOneBackup() throws Throwable { - } } -- 1.9.5.msysgit.0