Details
-
Bug
-
Status: Resolved
-
Normal
-
Resolution: Fixed
-
None
-
Correctness
-
Normal
-
Normal
-
User Report
-
All
-
None
-
Description
Fails 2 out of 10 times on Mac/Java 8
================================================================================================================== FAILURES ================================================================================================================== *___________________________________________________________________________________________________ TestBootstrap.test_resumable_bootstrap ___________________________________________________________________________________________________* self = <bootstrap_test.TestBootstrap object at 0x10fd488d0> *@since('2.2')* *def test_resumable_bootstrap(self):* *"""* *Test resuming bootstrap after data streaming failure* *"""* *cluster = self.cluster* *cluster.populate(2)* ** *node1 = cluster.nodes['node1']* *# set up byteman* *node1.byteman_port = '8100'* *node1.import_config_files()* ** *cluster.start(wait_other_notice=True)* *# kill stream to node3 in the middle of streaming to let it fail* *if cluster.version() < '4.0':* *node1.byteman_submit([self.byteman_submit_path_pre_4_0])* *else:* *node1.byteman_submit([self.byteman_submit_path_4_0])* *node1.stress(['write', 'n=1K', 'no-warmup', 'cl=TWO', '-schema', 'replication(factor=2)', '-rate', 'threads=50'])* *cluster.flush()* ** *# start bootstrapping node3 and wait for streaming* *node3 = new_node(cluster)* *node3.start(wait_other_notice=False)* ** *# let streaming fail as we expect* *> node3.watch_log_for('Some data streaming failed')* *bootstrap_test.py*:365: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ccmlib.node.Node object at 0x10fdf1810>, exprs = 'Some data streaming failed', from_mark = None, timeout = 600, process = None, verbose = False, filename = 'system.log' *def watch_log_for(self, exprs, from_mark=None, timeout=600, process=None, verbose=False, filename='system.log'):* *"""* *Watch the log until one or more (regular) expression are found.* *This methods when all the expressions have been found or the method* *timeouts (a TimeoutError is then raised). On successful completion,* *a list of pair (line matched, match object) is returned.* *"""* *start = time.time()* *tofind = [exprs] if isinstance(exprs, string_types) else exprs* *tofind = [re.compile(e) for e in tofind]* *matchings = []* *reads = ""* *if len(tofind) == 0:* *return None* ** *log_file = os.path.join(self.get_path(), 'logs', filename)* *output_read = False* *while not os.path.exists(log_file):* *time.sleep(.5)* *if start + timeout < time.time():* *raise TimeoutError(time.strftime("%d %b %Y %H:%M:%S", time.gmtime()) + " [" + self.name + "] Timed out waiting for {} to be created.".format(log_file))* *if process and not output_read:* *process.poll()* *if process.returncode is not None:* *self.print_process_output(self.name, process, verbose)* *output_read = True* *if process.returncode != 0:* *raise RuntimeError() # Shouldn't reuse RuntimeError but I'm lazy* ** *with open(log_file) as f:* *if from_mark:* *f.seek(from_mark)* ** *while True:* *# First, if we have a process to check, then check it.* *# Skip on Windows - stdout/stderr is cassandra.bat* *if not common.is_win() and not output_read:* *if process:* *process.poll()* *if process.returncode is not None:* *self.print_process_output(self.name, process, verbose)* *output_read = True* *if process.returncode != 0:* *raise RuntimeError() # Shouldn't reuse RuntimeError but I'm lazy* ** *line = f.readline()* *if line:* *reads = reads + line* *for e in tofind:* *m = e.search(line)* *if m:* *matchings.append((line, m))* *tofind.remove(e)* *if len(tofind) == 0:* *return matchings[0] if isinstance(exprs, string_types) else matchings* *else:* *# yep, it's ugly* *time.sleep(1)* *if start + timeout < time.time():* *> raise TimeoutError(time.strftime("%d %b %Y %H:%M:%S", time.gmtime()) + " [" + self.name + "] Missing: " + str([e.pattern for e in tofind]) + ":\n" + reads[:50] + ".....\nSee {} for remainder".format(filename))* *E ccmlib.node.TimeoutError: 02 Mar 2020 17:16:34 [node3] Missing: ['Some data streaming failed']:* *E INFO [main] 2020-03-02 12:06:34,963 YamlConfigura.....* *E See system.log for remainder* *../../dtest-new2/src/ccm/ccmlib/node.py*:536: TimeoutError ----------------------------------------------------------------------------------------------------------- Captured stdout setup ------------------------------------------------------------------------------------------------------------ 11:55:37,343 ccm DEBUG Log-watching thread starting. ------------------------------------------------------------------------------------------------------------- Captured log setup ------------------------------------------------------------------------------------------------------------- 11:55:37,239 conftest INFO Starting execution of test_resumable_bootstrap at 2020-03-02 11:55:37.239467 11:55:37,240 dtest_setup INFO cluster ccm directory: /var/folders/ql/nvcz74bd67d3vhw7227mpjm40000gp/T/dtest-fzxwwt9c ------------------------------------------------------------------------------------------------------------ Captured stdout call ------------------------------------------------------------------------------------------------------------ install rule inject stream failure ---------------------------------------------------------------------------------------------------------- Captured stdout teardown ---------------------------------------------------------------------------------------------------------- 12:06:04,870 ccm DEBUG Log-watching thread exiting. ----------------------------------------------------------------------------------------------------------- Captured stdout setup ------------------------------------------------------------------------------------------------------------ 12:06:06,20 ccm DEBUG Log-watching thread starting. ------------------------------------------------------------------------------------------------------------- Captured log setup ------------------------------------------------------------------------------------------------------------- 12:06:05,887 conftest INFO Starting execution of test_resumable_bootstrap at 2020-03-02 12:06:05.887592 12:06:05,888 dtest_setup INFO cluster ccm directory: /var/folders/ql/nvcz74bd67d3vhw7227mpjm40000gp/T/dtest-yntcltkx ------------------------------------------------------------------------------------------------------------ Captured stdout call ------------------------------------------------------------------------------------------------------------ install rule inject stream failure ---------------------------------------------------------------------------------------------------------- Captured stdout teardown ---------------------------------------------------------------------------------------------------------- 12:06:04,870 ccm DEBUG Log-watching thread exiting. ---------------------------------------------------------------------------------------------------------- Captured stdout teardown ---------------------------------------------------------------------------------------------------------- 12:06:04,870 ccm DEBUG Log-watching thread exiting. ---------------------------------------------------------------------------------------------------------- Captured stdout teardown ---------------------------------------------------------------------------------------------------------- 12:16:34,818 ccm DEBUG Log-watching thread exiting. ===Flaky Test Report=== test_resumable_bootstrap failed (1 runs remaining out of 2). <class 'ccmlib.node.TimeoutError'> 02 Mar 2020 17:06:04 [node3] Missing: ['Some data streaming failed']: INFO [main] 2020-03-02 11:56:05,081 YamlConfigura..... See system.log for remainder [<TracebackEntry /Users/ekaterina.dimitri/IdeaProjects/cassandra-dtest-d/bootstrap_test.py:365>, <TracebackEntry /Users/ekaterina.dimitri/dtest-new2/src/ccm/ccmlib/node.py:536>] test_resumable_bootstrap failed; it passed 0 out of the required 1 times. <class 'ccmlib.node.TimeoutError'> 02 Mar 2020 17:16:34 [node3] Missing: ['Some data streaming failed']: INFO [main] 2020-03-02 12:06:34,963 YamlConfigura..... See system.log for remainder [<TracebackEntry /Users/ekaterina.dimitri/IdeaProjects/cassandra-dtest-d/bootstrap_test.py:365>, <TracebackEntry /Users/ekaterina.dimitri/dtest-new2/src/ccm/ccmlib/node.py:536>] ===End Flaky Test Report=== *======================================================================================================== 1 failed in 1258.32 seconds =========================================================================================================*