Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136256756; first failure: 0; threshold: 10 TT-1 difference: 1467136256746 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136256762; first failure: 1467136256759; threshold: 10 TT-2 difference: -7 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136256803; first failure: 1467136256759; threshold: 10 TT-1 difference: 34 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136256805; first failure: 1467136256759; threshold: 10 TT-2 difference: 36 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136256843; first failure: 1467136256759; threshold: 10 TT-1 difference: 74 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136256900; first failure: 1467136256759; threshold: 10 TT-1 difference: 131 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136256908; first failure: 1467136256759; threshold: 10 TT-1 difference: 139 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136257017; first failure: 0; threshold: 10 TT-1 difference: 1467136257007 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136257018; first failure: 1467136257017; threshold: 10 TT-2 difference: -9 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136257040; first failure: 1467136257017; threshold: 10 TT-1 difference: 13 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136257046; first failure: 1467136257017; threshold: 10 TT-2 difference: 19 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257078; first failure: 1467136257017; threshold: 10 TT-1 difference: 51 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257129; first failure: 1467136257017; threshold: 10 TT-1 difference: 102 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257132; first failure: 1467136257017; threshold: 10 TT-1 difference: 105 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136257335; first failure: 0; threshold: 10 TT-1 difference: 1467136257325 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136257337; first failure: 1467136257336; threshold: 10 TT-2 difference: -9 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136257375; first failure: 1467136257336; threshold: 10 TT-1 difference: 29 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136257376; first failure: 1467136257336; threshold: 10 TT-2 difference: 30 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257409; first failure: 1467136257336; threshold: 10 TT-1 difference: 63 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257460; first failure: 1467136257336; threshold: 10 TT-1 difference: 114 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257461; first failure: 1467136257336; threshold: 10 TT-1 difference: 115 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136257491; first failure: 0; threshold: 10 TT-1 difference: 1467136257481 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136257491; first failure: 1467136257491; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136257514; first failure: 1467136257491; threshold: 10 TT-1 difference: 13 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136257515; first failure: 1467136257491; threshold: 10 TT-2 difference: 14 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257552; first failure: 1467136257491; threshold: 10 TT-1 difference: 51 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257602; first failure: 1467136257491; threshold: 10 TT-1 difference: 101 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257604; first failure: 1467136257491; threshold: 10 TT-1 difference: 103 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136257620; first failure: 0; threshold: 10 TT-1 difference: 1467136257610 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136257621; first failure: 1467136257620; threshold: 10 TT-2 difference: -9 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136257646; first failure: 1467136257620; threshold: 10 TT-1 difference: 16 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136257648; first failure: 1467136257620; threshold: 10 TT-2 difference: 18 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257685; first failure: 1467136257620; threshold: 10 TT-1 difference: 55 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257739; first failure: 1467136257620; threshold: 10 TT-1 difference: 109 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257739; first failure: 1467136257620; threshold: 10 TT-1 difference: 109 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136257777; first failure: 0; threshold: 10 TT-1 difference: 1467136257767 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136257777; first failure: 1467136257777; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136257814; first failure: 1467136257777; threshold: 10 TT-1 difference: 27 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136257814; first failure: 1467136257777; threshold: 10 TT-2 difference: 27 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257846; first failure: 1467136257777; threshold: 10 TT-1 difference: 59 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257903; first failure: 1467136257777; threshold: 10 TT-1 difference: 116 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257907; first failure: 1467136257777; threshold: 10 TT-1 difference: 120 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136257934; first failure: 0; threshold: 10 TT-1 difference: 1467136257924 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136257934; first failure: 1467136257934; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136257957; first failure: 1467136257934; threshold: 10 TT-1 difference: 13 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136257963; first failure: 1467136257934; threshold: 10 TT-2 difference: 19 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136257995; first failure: 1467136257934; threshold: 10 TT-1 difference: 51 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258050; first failure: 1467136257934; threshold: 10 TT-1 difference: 106 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258052; first failure: 1467136257934; threshold: 10 TT-1 difference: 108 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136258074; first failure: 0; threshold: 10 TT-1 difference: 1467136258064 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136258074; first failure: 1467136258074; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136258098; first failure: 1467136258074; threshold: 10 TT-1 difference: 14 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136258100; first failure: 1467136258074; threshold: 10 TT-2 difference: 16 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258132; first failure: 1467136258074; threshold: 10 TT-1 difference: 48 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258183; first failure: 1467136258074; threshold: 10 TT-1 difference: 99 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258183; first failure: 1467136258074; threshold: 10 TT-1 difference: 99 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136258203; first failure: 0; threshold: 10 TT-1 difference: 1467136258193 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136258210; first failure: 1467136258209; threshold: 10 TT-2 difference: -9 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136258257; first failure: 1467136258209; threshold: 10 TT-1 difference: 38 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136258260; first failure: 1467136258209; threshold: 10 TT-2 difference: 41 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258291; first failure: 1467136258209; threshold: 10 TT-1 difference: 72 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258343; first failure: 1467136258209; threshold: 10 TT-1 difference: 124 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258344; first failure: 1467136258209; threshold: 10 TT-1 difference: 125 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136258360; first failure: 0; threshold: 10 TT-1 difference: 1467136258350 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136258360; first failure: 1467136258360; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136258381; first failure: 1467136258360; threshold: 10 TT-1 difference: 11 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136258381; first failure: 1467136258360; threshold: 10 TT-2 difference: 11 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258412; first failure: 1467136258360; threshold: 10 TT-1 difference: 42 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258464; first failure: 1467136258360; threshold: 10 TT-1 difference: 94 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258465; first failure: 1467136258360; threshold: 10 TT-1 difference: 95 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136258524; first failure: 0; threshold: 10 TT-1 difference: 1467136258514 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136258524; first failure: 1467136258524; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136258546; first failure: 1467136258524; threshold: 10 TT-1 difference: 12 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136258548; first failure: 1467136258524; threshold: 10 TT-2 difference: 14 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258578; first failure: 1467136258524; threshold: 10 TT-1 difference: 44 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258630; first failure: 1467136258524; threshold: 10 TT-1 difference: 96 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258630; first failure: 1467136258524; threshold: 10 TT-1 difference: 96 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136258644; first failure: 0; threshold: 10 TT-1 difference: 1467136258634 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136258644; first failure: 1467136258644; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136258665; first failure: 1467136258644; threshold: 10 TT-1 difference: 11 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136258666; first failure: 1467136258644; threshold: 10 TT-2 difference: 12 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258697; first failure: 1467136258644; threshold: 10 TT-1 difference: 43 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258748; first failure: 1467136258644; threshold: 10 TT-1 difference: 94 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258748; first failure: 1467136258644; threshold: 10 TT-1 difference: 94 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-2 L1[0].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136258763; first failure: 0; threshold: 10 TT-1 difference: 1467136258753 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136258763; first failure: 1467136258763; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258785; first failure: 1467136258763; threshold: 10 TT-1 difference: 12 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136258786; first failure: 1467136258763; threshold: 10 TT-2 difference: 13 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258817; first failure: 1467136258763; threshold: 10 TT-1 difference: 44 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258868; first failure: 1467136258763; threshold: 10 TT-1 difference: 95 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258870; first failure: 1467136258763; threshold: 10 TT-1 difference: 97 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136258902; first failure: 0; threshold: 10 TT-1 difference: 1467136258892 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136258903; first failure: 1467136258903; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136258936; first failure: 1467136258903; threshold: 10 TT-1 difference: 23 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136258937; first failure: 1467136258903; threshold: 10 TT-2 difference: 24 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136258968; first failure: 1467136258903; threshold: 10 TT-1 difference: 55 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259018; first failure: 1467136258903; threshold: 10 TT-1 difference: 105 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259019; first failure: 1467136258903; threshold: 10 TT-1 difference: 106 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136259032; first failure: 0; threshold: 10 TT-1 difference: 1467136259022 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136259032; first failure: 1467136259032; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136259054; first failure: 1467136259032; threshold: 10 TT-1 difference: 12 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136259055; first failure: 1467136259032; threshold: 10 TT-2 difference: 13 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259086; first failure: 1467136259032; threshold: 10 TT-1 difference: 44 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259137; first failure: 1467136259032; threshold: 10 TT-1 difference: 95 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259137; first failure: 1467136259032; threshold: 10 TT-1 difference: 95 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136259151; first failure: 0; threshold: 10 TT-1 difference: 1467136259141 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136259151; first failure: 1467136259151; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136259173; first failure: 1467136259151; threshold: 10 TT-1 difference: 12 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136259175; first failure: 1467136259151; threshold: 10 TT-2 difference: 14 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259207; first failure: 1467136259151; threshold: 10 TT-1 difference: 46 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259258; first failure: 1467136259151; threshold: 10 TT-1 difference: 97 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259259; first failure: 1467136259151; threshold: 10 TT-1 difference: 98 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-2 L1[0].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136259275; first failure: 0; threshold: 10 TT-1 difference: 1467136259265 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136259276; first failure: 1467136259275; threshold: 10 TT-2 difference: -9 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136259297; first failure: 1467136259275; threshold: 10 TT-1 difference: 12 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136259297; first failure: 1467136259275; threshold: 10 TT-2 difference: 12 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259329; first failure: 1467136259275; threshold: 10 TT-1 difference: 44 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259380; first failure: 1467136259275; threshold: 10 TT-1 difference: 95 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259382; first failure: 1467136259275; threshold: 10 TT-1 difference: 97 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136259408; first failure: 0; threshold: 10 TT-1 difference: 1467136259398 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136259409; first failure: 1467136259408; threshold: 10 TT-2 difference: -9 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136259441; first failure: 1467136259408; threshold: 10 TT-1 difference: 23 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136259443; first failure: 1467136259408; threshold: 10 TT-2 difference: 25 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259477; first failure: 1467136259408; threshold: 10 TT-1 difference: 59 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259529; first failure: 1467136259408; threshold: 10 TT-1 difference: 111 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259530; first failure: 1467136259408; threshold: 10 TT-1 difference: 112 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136259574; first failure: 0; threshold: 10 TT-1 difference: 1467136259564 TT-1 L1[0].countDown(); handleFailure() TT-2 L1[0].await(); intercept() TT-1 pffe: false; updateFailureInfo() TT-2 intercept#inFastFail(); TT-1 L2[1].await(); updateFailureInfo() TT-2 current: 1467136259575; first failure: 1467136259575; threshold: 10 TT-2 difference: -10 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136259620; first failure: 1467136259575; threshold: 10 TT-1 difference: 35 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136259633; first failure: 1467136259575; threshold: 10 TT-2 difference: 48 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259680; first failure: 1467136259575; threshold: 10 TT-1 difference: 95 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259732; first failure: 1467136259575; threshold: 10 TT-1 difference: 147 TT-1 L1[3].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259733; first failure: 1467136259575; threshold: 10 TT-1 difference: 148 TT-1 pffe: false; updateFailureInfo() Time-limited test ====================================================================== TT-1 intercept#inFastFail(); TT-1 current: 1467136259783; first failure: 0; threshold: 10 TT-1 difference: 1467136259773 TT-1 L1[0].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[1].await(); updateFailureInfo() TT-2 L1[0].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136259797; first failure: 1467136259783; threshold: 10 TT-2 difference: 4 TT-2 pffe: false; updateFailureInfo() TT-2 pffe: false, done: 0, increment: done=0 TT-2 L2[1].countDown(); updateFailureInfo() TT-2 L1[1].await(); intercept() TT-1 intercept#inFastFail(); TT-1 current: 1467136259819; first failure: 1467136259783; threshold: 10 TT-1 difference: 26 TT-1 L1[1].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[2].await(); updateFailureInfo() TT-2 intercept#inFastFail(); TT-2 current: 1467136259819; first failure: 1467136259783; threshold: 10 TT-2 difference: 26 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 0, increment: done = 1 TT-2 L2[2].countDown(); updateFailureInfo() TT-1 intercept#inFastFail(); TT-1 current: 1467136259856; first failure: 1467136259783; threshold: 10 TT-1 difference: 63 TT-1 L1[2].countDown(); handleFailure() TT-1 pffe: false; updateFailureInfo() TT-1 L2[3].await(); updateFailureInfo() Process Thread Dump: This should not hang but seems to sometimes...FIX! Here is a thread dump! 9 active threads Thread 14 (TT-2): State: TIMED_WAITING Blocked count: 2 Waited count: 61 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) Thread 13 (TT-1): State: WAITING Blocked count: 3 Waited count: 138 Waiting on java.util.concurrent.CountDownLatch$Sync@41474389 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$7.updateFailureInfo(TestFastFailWithoutTestUtil.java:546) org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:138) org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$8.callWithRetries(TestFastFailWithoutTestUtil.java:594) org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$8.callWithRetries(TestFastFailWithoutTestUtil.java:590) org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$3.call(TestFastFailWithoutTestUtil.java:421) org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$3.call(TestFastFailWithoutTestUtil.java:415) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) Thread 11 (process reaper): State: TIMED_WAITING Blocked count: 0 Waited count: 1 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) Thread 9 (Time-limited test): State: RUNNABLE Blocked count: 1 Waited count: 60 Stack: sun.management.ThreadImpl.getThreadInfo1(Native Method) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:178) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:139) org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.java:168) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) org.apache.hadoop.hbase.util.Threads$PrintThreadInfoLazyHolder$1.printThreadInfo(Threads.java:294) org.apache.hadoop.hbase.util.Threads.printThreadInfo(Threads.java:340) org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil.testPreemptiveFastFailException(TestFastFailWithoutTestUtil.java:463) org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil.testPreemptiveFastFailException50Times(TestFastFailWithoutTestUtil.java:337) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) Thread 8 (Monitor Ctrl-Break): State: RUNNABLE Blocked count: 1 Waited count: 0 Stack: java.net.PlainSocketImpl.socketAccept(Native Method) java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) java.net.ServerSocket.implAccept(ServerSocket.java:545) java.net.ServerSocket.accept(ServerSocket.java:513) com.intellij.rt.execution.application.AppMain$1.run(AppMain.java:90) java.lang.Thread.run(Thread.java:745) Thread 4 (Signal Dispatcher): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: Thread 3 (Finalizer): State: WAITING Blocked count: 4 Waited count: 5 Waiting on java.lang.ref.ReferenceQueue$Lock@5a3cc4db Stack: java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Thread 2 (Reference Handler): State: WAITING Blocked count: 3 Waited count: 4 Waiting on java.lang.ref.Reference$Lock@2cc788f4 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) java.lang.ref.Reference.tryHandlePending(Reference.java:191) java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Thread 1 (main): State: TIMED_WAITING Blocked count: 0 Waited count: 2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.FutureTask.awaitDone(FutureTask.java:426) java.util.concurrent.FutureTask.get(FutureTask.java:204) org.junit.internal.runners.statements.FailOnTimeout.getResult(FailOnTimeout.java:141) org.junit.internal.runners.statements.FailOnTimeout.evaluate(FailOnTimeout.java:127) org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) org.junit.runners.ParentRunner.run(ParentRunner.java:363) org.junit.runner.JUnitCore.run(JUnitCore.java:137) com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:119) com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:42) com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:234) com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:74) TT-2 L1[2].await(); intercept() TT-2 intercept#inFastFail(); TT-2 current: 1467136289853; first failure: 1467136259783; threshold: 10 TT-2 difference: 30060 TT-2 pffe: true; updateFailureInfo() TT-2 pffe: true, done: 1, increment: done = 2 TT-2 L2[3].countDown(); updateFailureInfo() java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException: Exception happened 4 times. tolocalhost,1234,987654321 at java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.util.concurrent.FutureTask.get(FutureTask.java:192) at org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil.testPreemptiveFastFailException(TestFastFailWithoutTestUtil.java:481) at org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil.testPreemptiveFastFailException50Times(TestFastFailWithoutTestUtil.java:337) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException: Exception happened 4 times. tolocalhost,1234,987654321 at org.apache.hadoop.hbase.client.PreemptiveFastFailInterceptor.intercept(PreemptiveFastFailInterceptor.java:124) at org.apache.hadoop.hbase.client.PreemptiveFastFailInterceptor.intercept(PreemptiveFastFailInterceptor.java:342) at org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$7.intercept(TestFastFailWithoutTestUtil.java:573) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:101) at org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$8.callWithRetries(TestFastFailWithoutTestUtil.java:594) at org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$8.callWithRetries(TestFastFailWithoutTestUtil.java:590) at org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$5.call(TestFastFailWithoutTestUtil.java:477) at org.apache.hadoop.hbase.client.TestFastFailWithoutTestUtil$5.call(TestFastFailWithoutTestUtil.java:474) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ... 1 more 2016-06-28 19:51:29,856 ERROR [TT-1] client.PreemptiveFastFailInterceptor(213): localhost,1234,987654321 been failing for a long time. clearing out.FailureInfo: numConsecutiveFailures = 4 timeOfFirstFailureMilliSec = 1467136259783 timeOfLatestAttemptMilliSec = 1467136289854 exclusivelyRetringInspiteOfFastFail = false