Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerCoprocessorException.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerCoprocessorException.java (revision 0) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerCoprocessorException.java (revision 0) @@ -0,0 +1,145 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFlushContext; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +/** + * Tests unhandled exceptions thrown by coprocessors running on regionserver. + * Expected result is that the region server will remove the buggy coprocessor from + * its set of coprocessors and throw a org.apache.hadoop.hbase.exceptions.DoNotRetryIOException + * back to the client. + * (HBASE-4014). + */ +@Category(MediumTests.class) +public class TestRegionServerCoprocessorException { + static byte[] TEST_FAMILY = Bytes.toBytes("aaa"); + public static class BuggyRegionObserver extends SimpleRegionObserver { + @SuppressWarnings("null") + @Override + public void postBatchMutate(final ObserverContext c, + final MiniBatchOperationInProgress miniBatchOp) throws IOException { + super.postBatchMutate(c, miniBatchOp); + String tableName = + c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString(); + if (tableName.equals("observed_table")) { + // trigger memstore snapshot + HRegion region = c.getEnvironment().getRegion(); + Store store = region.getStore(TEST_FAMILY); + StoreFlushContext storeFlushCtx = store.createFlushContext(12345); + storeFlushCtx.prepare(); + + // Trigger a NPE to fail the coprocessor + Integer i = null; + i = i + 1; + } + } + } + + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // set configure to indicate which cp should be loaded + Configuration conf = TEST_UTIL.getConfiguration(); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + BuggyRegionObserver.class.getName()); + TEST_UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false); + TEST_UTIL.startMiniCluster(); + } + + @AfterClass + public static void teardownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test(timeout=90000) + public void testExceptionFromCoprocessorDuringPostBatchMutate() + throws IOException, InterruptedException { + // Set watches on the zookeeper nodes for all of the regionservers in the + // cluster. When we try to write to TEST_TABLE, the buggy coprocessor will + // cause a NullPointerException, which will cause the regionserver (which + // hosts the region we attempted to write to) to abort. In turn, this will + // cause the nodeDeleted() method of the DeadRegionServer tracker to + // execute, which will set the rsZKNodeDeleted flag to true, which will + // pass this test. + + TableName TEST_TABLE = + TableName.valueOf("observed_table"); + + HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY); + TEST_UTIL.createMultiRegions(table, TEST_FAMILY); + TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE); + // Note which regionServer that should survive the buggy coprocessor's + // prePut(). + HRegionServer regionServer = + TEST_UTIL.getRSForFirstRegionInTable(TEST_TABLE); + HRegion region = null; + byte[] ROW = null; + for (HRegion r : regionServer.getOnlineRegions(TEST_TABLE)) { + ROW = r.getStartKey(); + region = r; + break; + } + + boolean threwIOE = false; + try { + Put put = new Put(ROW); + put.add(TEST_FAMILY, ROW, ROW); + table.put(put); + } catch (IOException e) { + threwIOE = true; + } finally { + assertTrue("The regionserver should have thrown an exception", threwIOE); + } + + Store store = region.getStore(TEST_FAMILY); + long sz = store.getFlushableSize(); + assertTrue("flushable size should be zero, but it is " + sz, sz == 0); + + table.close(); + } + +} + Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java (revision 1582182) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java (working copy) @@ -291,6 +291,8 @@ KeyValue found = this.snapshot.get(kv); if (found != null && found.getMvccVersion() == kv.getMvccVersion()) { this.snapshot.remove(kv); + long sz = heapSizeChange(kv, true); + this.snapshotSize -= sz; } // If the key is in the memstore, delete it. Update this.size. found = this.kvset.get(kv); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (revision 1582182) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (working copy) @@ -2486,7 +2486,6 @@ if (hasWalAppends) { syncOrDefer(txid, durability); } - doRollBackMemstore = false; // calling the post CP hook for batch mutation if (!isInReplay && coprocessorHost != null) { MiniBatchOperationInProgress miniBatchOp = @@ -2494,6 +2493,7 @@ batchOp.retCodeDetails, batchOp.walEditsFromCoprocessors, firstIndex, lastIndexExclusive); coprocessorHost.postBatchMutate(miniBatchOp); } + doRollBackMemstore = false; // ------------------------------------------------------------------ // STEP 8. Advance mvcc. This will make this put visible to scanners and getters.