commit 16ccb8be6771ef57537dbb9e3d0abc4bf7caa3ba Author: Todd Lipcon Date: Fri May 21 14:39:40 2010 -0700 HBASE-2519. Expose FS level errors through scanners diff --git src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java index 70b85af..2fff71e 100644 --- src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java +++ src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java @@ -80,15 +80,15 @@ public class ScannerCallable extends ServerCallable { if (e instanceof RemoteException) { ioe = RemoteExceptionHandler.decodeRemoteException((RemoteException)e); } - if (ioe != null) { - if (ioe instanceof NotServingRegionException) { + if (ioe == null) throw new IOException(e); + if (ioe instanceof NotServingRegionException) { // Throw a DNRE so that we break out of cycle of calling NSRE // when what we need is to open scanner against new location. // Attach NSRE to signal client that it needs to resetup scanner. throw new DoNotRetryIOException("Reset scanner", ioe); - } else if (ioe instanceof DoNotRetryIOException) { - throw ioe; - } + } else { + // The outer layers will retry + throw ioe; } } return rrs; diff --git src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 3433811..cf4768f 100644 --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -1368,6 +1368,11 @@ public class HFile { } } } + + @Override + public String toString() { + return "HFileScanner for reader " + String.valueOf(reader); + } } public String getTrailerInfo() { diff --git src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 038a335..ff9e278 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1952,7 +1952,7 @@ public class HRegion implements HConstants, HeapSize { // , Writable{ this(scan, null); } - void initHeap() { + void initHeap() throws IOException { List scanners = new ArrayList(); if (extraScanners != null) { scanners.addAll(extraScanners); diff --git src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java index 70f42dc..b0c7e95 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java @@ -72,7 +72,7 @@ public class KeyValueHeap implements KeyValueScanner, InternalScanner { return this.current.peek(); } - public KeyValue next() { + public KeyValue next() throws IOException { if(this.current == null) { return null; } @@ -178,8 +178,9 @@ public class KeyValueHeap implements KeyValueScanner, InternalScanner { * automatically closed and removed from the heap. * @param seekKey KeyValue to seek at or after * @return true if KeyValues exist at or after specified key, false if not + * @throws IOException */ - public boolean seek(KeyValue seekKey) { + public boolean seek(KeyValue seekKey) throws IOException { if(this.current == null) { return false; } diff --git src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java index 657018f..da6e450 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java @@ -19,6 +19,8 @@ */ package org.apache.hadoop.hbase.regionserver; +import java.io.IOException; + import org.apache.hadoop.hbase.KeyValue; /** @@ -35,17 +37,17 @@ public interface KeyValueScanner { * Return the next KeyValue in this scanner, iterating the scanner * @return the next KeyValue */ - public KeyValue next(); + public KeyValue next() throws IOException; /** * Seek the scanner at or after the specified KeyValue. * @param key seek value * @return true if scanner has values left, false if end of scanner */ - public boolean seek(KeyValue key); + public boolean seek(KeyValue key) throws IOException; /** * Close the KeyValue scanner. */ public void close(); -} \ No newline at end of file +} diff --git src/main/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java src/main/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java index 4b16540..71f738e 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java @@ -36,7 +36,8 @@ public class MinorCompactingStoreScanner implements KeyValueScanner, InternalSca private KeyValueHeap heap; private KeyValue.KVComparator comparator; - MinorCompactingStoreScanner(Store store, List scanners) { + MinorCompactingStoreScanner(Store store, List scanners) + throws IOException { comparator = store.comparator; KeyValue firstKv = KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW); for (KeyValueScanner scanner : scanners ) { @@ -46,7 +47,8 @@ public class MinorCompactingStoreScanner implements KeyValueScanner, InternalSca } MinorCompactingStoreScanner(String cfName, KeyValue.KVComparator comparator, - List scanners) { + List scanners) + throws IOException { this.comparator = comparator; KeyValue firstKv = KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW); @@ -61,7 +63,7 @@ public class MinorCompactingStoreScanner implements KeyValueScanner, InternalSca return heap.peek(); } - public KeyValue next() { + public KeyValue next() throws IOException { return heap.next(); } diff --git src/main/java/org/apache/hadoop/hbase/regionserver/Store.java src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index 6c3153b..7de766d 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -1304,9 +1304,10 @@ public class Store implements HConstants, HeapSize { /** * Return a scanner for both the memstore and the HStore files + * @throws IOException */ protected KeyValueScanner getScanner(Scan scan, - final NavigableSet targetCols) { + final NavigableSet targetCols) throws IOException { lock.readLock().lock(); try { return new StoreScanner(this, scan, targetCols); diff --git src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java index 52d228b..3541b3b 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java @@ -56,17 +56,11 @@ class StoreFileScanner implements KeyValueScanner { public static List getScannersForStoreFiles( Collection filesToCompact, boolean cacheBlocks, - boolean usePread) { + boolean usePread) throws IOException { List scanners = new ArrayList(filesToCompact.size()); for (StoreFile file : filesToCompact) { - Reader r = file.getReader(); - if (r == null) { - // TODO why can this happen? this seems like something worth - // throwing an exception over! - LOG.error("StoreFile " + file + " has a null Reader"); - continue; - } + Reader r = file.createReader(); scanners.add(new StoreFileScanner(r.getScanner(cacheBlocks, usePread))); } return scanners; @@ -84,7 +78,7 @@ class StoreFileScanner implements KeyValueScanner { return cur; } - public KeyValue next() { + public KeyValue next() throws IOException { KeyValue retKey = cur; cur = hfs.getKeyValue(); try { @@ -92,13 +86,12 @@ class StoreFileScanner implements KeyValueScanner { if (cur != null) hfs.next(); } catch(IOException e) { - // Turn checked exception into runtime exception. - throw new RuntimeException(e); + throw new IOException("Could not iterate " + this, e); } return retKey; } - public boolean seek(KeyValue key) { + public boolean seek(KeyValue key) throws IOException { try { if(!seekAtOrAfter(hfs, key)) { close(); @@ -108,8 +101,7 @@ class StoreFileScanner implements KeyValueScanner { hfs.next(); return true; } catch(IOException ioe) { - close(); - return false; + throw new IOException("Could not seek " + this, ioe); } } diff --git src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index fde872c..c8941f1 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -52,8 +52,9 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb * @param store who we scan * @param scan the spec * @param columns which columns we are scanning + * @throws IOException */ - StoreScanner(Store store, Scan scan, final NavigableSet columns) { + StoreScanner(Store store, Scan scan, final NavigableSet columns) throws IOException { this.store = store; this.cacheBlocks = scan.getCacheBlocks(); matcher = new ScanQueryMatcher(scan, store.getFamily().getName(), @@ -83,7 +84,8 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb * @param scan the spec * @param scanners ancilliary scanners */ - StoreScanner(Store store, Scan scan, List scanners) { + StoreScanner(Store store, Scan scan, List scanners) + throws IOException { this.store = store; this.cacheBlocks = false; this.isGet = false; @@ -104,7 +106,8 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb StoreScanner(final Scan scan, final byte [] colFamily, final long ttl, final KeyValue.KVComparator comparator, final NavigableSet columns, - final List scanners) { + final List scanners) + throws IOException { this.store = null; this.isGet = false; this.cacheBlocks = scan.getCacheBlocks(); @@ -121,7 +124,7 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb /* * @return List of scanners ordered properly. */ - private List getScanners() { + private List getScanners() throws IOException { // First the store file scanners Map map = this.store.getStorefiles().descendingMap(); List sfScanners = StoreFileScanner @@ -138,7 +141,7 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb * @return List of scanners to seek, possibly filtered by StoreFile. */ private List getScanners(Scan scan, - final NavigableSet columns) { + final NavigableSet columns) throws IOException { // First the store file scanners Map map = this.store.getStorefiles().descendingMap(); List sfScanners = StoreFileScanner @@ -178,7 +181,7 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb this.heap.close(); } - public synchronized boolean seek(KeyValue key) { + public synchronized boolean seek(KeyValue key) throws IOException { return this.heap.seek(key); } @@ -282,4 +285,4 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb KeyValue kv = heap.peek(); matcher.setRow((kv == null ? topKey : kv).getRow()); } -} \ No newline at end of file +} diff --git src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 238e804..ed8709f 100644 --- src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -471,6 +471,21 @@ public class HBaseTestingUtility { } /** + * Return the number of rows in the given table. + */ + public int countRows(final HTable table) throws IOException { + Scan scan = new Scan(); + ResultScanner results = table.getScanner(scan); + int count = 0; + for (@SuppressWarnings("unused") Result res : results) { + count++; + } + results.close(); + return count; + } + + + /** * Creates many regions names "aaa" to "zzz". * * @param table The table to use for the data. @@ -727,4 +742,8 @@ public class HBaseTestingUtility { public MiniDFSCluster getDFSCluster() { return dfsCluster; } -} \ No newline at end of file + + public FileSystem getTestFileSystem() throws IOException { + return FileSystem.get(conf); + } +} diff --git src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java index 55a926f..441bb3c 100644 --- src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java +++ src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hdfs.DistributedFileSystem; @@ -345,6 +346,19 @@ public class MiniHBaseCluster implements HConstants { public HRegionServer getRegionServer(int serverNumber) { return hbaseCluster.getRegionServer(serverNumber); } + + public List getRegions(byte[] tableName) { + List ret = new ArrayList(); + for (JVMClusterUtil.RegionServerThread rst : getRegionServerThreads()) { + HRegionServer hrs = rst.getRegionServer(); + for (HRegion region : hrs.getOnlineRegions()) { + if (Bytes.equals(region.getTableDesc().getName(), tableName)) { + ret.add(region); + } + } + } + return ret; + } /** * @return Index into List of {@link MiniHBaseCluster#getRegionServerThreads()} diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java new file mode 100644 index 0000000..59537c8 --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java @@ -0,0 +1,233 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.ref.SoftReference; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + + +/** + * Test cases that ensure that file system level errors are bubbled up + * appropriately to clients, rather than swallowed. + */ +public class TestFSErrorsExposed { + private static final Log LOG = LogFactory.getLog(TestFSErrorsExposed.class); + + HBaseTestingUtility util = new HBaseTestingUtility(); + + /** + * Injects errors into the pread calls of an on-disk file, and makes + * sure those bubble up to the HFile scanner + */ + @Test + public void testHFileScannerThrowsErrors() throws IOException { + Path hfilePath = new Path(new Path( + HBaseTestingUtility.getTestDir("internalScannerExposesErrors"), + "regionname"), "familyname"); + FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem()); + StoreFile.Writer writer = StoreFile.createWriter(fs, hfilePath, 2*1024); + TestStoreFile.writeStoreFile( + writer, Bytes.toBytes("cf"), Bytes.toBytes("qual")); + + StoreFile sf = new StoreFile(fs, writer.getPath(), false, + util.getConfiguration(), StoreFile.BloomType.NONE, false); + HFile.Reader reader = sf.createReader(); + HFileScanner scanner = reader.getScanner(false, true); + + FaultyInputStream inStream = fs.inStreams.get(0).get(); + assertNotNull(inStream); + + scanner.seekTo(); + // Do at least one successful read + assertTrue(scanner.next()); + + inStream.startFaults(); + + try { + int scanned=0; + while (scanner.next()) { + scanned++; + } + fail("Scanner didn't throw after faults injected"); + } catch (IOException ioe) { + LOG.info("Got expected exception", ioe); + assertTrue(ioe.getMessage().contains("Fault")); + } + reader.close(); + } + + /** + * Injects errors into the pread calls of an on-disk file, and makes + * sure those bubble up to the StoreFileScanner + */ + @Test + public void testStoreFileScannerThrowsErrors() throws IOException { + Path hfilePath = new Path(new Path( + HBaseTestingUtility.getTestDir("internalScannerExposesErrors"), + "regionname"), "familyname"); + FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem()); + HFile.Writer writer = StoreFile.createWriter(fs, hfilePath, 2 * 1024); + TestStoreFile.writeStoreFile( + writer, Bytes.toBytes("cf"), Bytes.toBytes("qual")); + + StoreFile sf = new StoreFile(fs, writer.getPath(), false, + util.getConfiguration(), BloomType.NONE, false); + List scanners = StoreFileScanner.getScannersForStoreFiles( + Collections.singletonList(sf), false, true); + KeyValueScanner scanner = scanners.get(0); + + FaultyInputStream inStream = fs.inStreams.get(0).get(); + assertNotNull(inStream); + + scanner.seek(KeyValue.LOWESTKEY); + // Do at least one successful read + assertNotNull(scanner.next()); + + inStream.startFaults(); + + try { + int scanned=0; + while (scanner.next() != null) { + scanned++; + } + fail("Scanner didn't throw after faults injected"); + } catch (IOException ioe) { + LOG.info("Got expected exception", ioe); + assertTrue(ioe.getMessage().contains("Could not iterate")); + } + scanner.close(); + } + + /** + * Cluster test which starts a region server with a region, then + * removes the data from HDFS underneath it, and ensures that + * errors are bubbled to the client. + */ + @Test + public void testFullSystemBubblesFSErrors() throws Exception { + try { + util.startMiniCluster(1); + byte[] tableName = Bytes.toBytes("table"); + byte[] fam = Bytes.toBytes("fam"); + + HBaseAdmin admin = new HBaseAdmin(util.getConfiguration()); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor( + fam, 1, HColumnDescriptor.DEFAULT_COMPRESSION, + false, false, HConstants.FOREVER, "NONE")); + admin.createTable(desc); + + HTable table = new HTable(tableName); + + // Load some data + util.loadTable(table, fam); + table.flushCommits(); + util.flush(); + util.countRows(table); + + // Kill the DFS cluster + util.getDFSCluster().shutdownDataNodes(); + + try { + util.countRows(table); + fail("Did not fail to count after removing data"); + } catch (RuntimeException rte) { + // We get RTE instead of IOE since java Iterable doesn't throw + // IOE + LOG.info("Got expected error", rte); + assertTrue(rte.getMessage().contains("Could not seek")); + } + + } finally { + util.shutdownMiniCluster(); + } + } + + static class FaultyFileSystem extends FilterFileSystem { + List> inStreams = + new ArrayList>(); + + public FaultyFileSystem(FileSystem testFileSystem) { + super(testFileSystem); + } + + @Override + public FSDataInputStream open(Path p, int bufferSize) throws IOException { + FSDataInputStream orig = fs.open(p, bufferSize); + FaultyInputStream faulty = new FaultyInputStream(orig); + inStreams.add(new SoftReference(faulty)); + return faulty; + } + } + + static class FaultyInputStream extends FSDataInputStream { + boolean faultsStarted = false; + + public FaultyInputStream(InputStream in) throws IOException { + super(in); + } + + public void startFaults() { + faultsStarted = true; + } + + public int read(long position, byte[] buffer, int offset, int length) + throws IOException { + injectFault(); + return ((PositionedReadable)in).read(position, buffer, offset, length); + } + + private void injectFault() throws IOException { + if (faultsStarted) { + throw new IOException("Fault injected"); + } + } + } + + +} diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java index 228ab2c..51f8829 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -64,7 +65,7 @@ implements HConstants { col5 = Bytes.toBytes("col5"); } - public void testSorted(){ + public void testSorted() throws IOException{ //Cases that need to be checked are: //1. The "smallest" KeyValue is in the same scanners as current //2. Current scanner gets empty @@ -126,7 +127,7 @@ implements HConstants { } - public void testSeek(){ + public void testSeek() throws IOException { //Cases: //1. Seek KeyValue that is not in scanner //2. Check that smallest that is returned from a seek is correct @@ -174,7 +175,7 @@ implements HConstants { } - public void testScannerLeak() { + public void testScannerLeak() throws IOException { // Test for unclosed scanners (HBASE-1927) List l1 = new ArrayList(); diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java index e1ffcc3..0ebeee4 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueScanFixture.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.regionserver; +import java.io.IOException; + import junit.framework.TestCase; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueTestUtil; @@ -28,7 +30,7 @@ import org.apache.hadoop.hbase.util.Bytes; public class TestKeyValueScanFixture extends TestCase { - public void testKeyValueScanFixture() { + public void testKeyValueScanFixture() throws IOException { KeyValue kvs[] = new KeyValue[]{ KeyValueTestUtil.create("RowA", "family", "qf1", 1, KeyValue.Type.Put, "value-1"), diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java index 6ed0209..9833d76 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java @@ -158,8 +158,9 @@ public class TestMemStore extends TestCase { /** * A simple test which verifies the 3 possible states when scanning across snapshot. + * @throws IOException */ - public void testScanAcrossSnapshot2() { + public void testScanAcrossSnapshot2() throws IOException { // we are going to the scanning across snapshot with two kvs // kv1 should always be returned before kv2 final byte[] one = Bytes.toBytes(1); @@ -188,7 +189,8 @@ public class TestMemStore extends TestCase { verifyScanAcrossSnapshot2(kv1, kv2); } - private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) { + private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2) + throws IOException { ReadWriteConsistencyControl.resetThreadReadPoint(rwcc); List memstorescanners = this.memstore.getScanners(); assertEquals(1, memstorescanners.size()); @@ -199,7 +201,8 @@ public class TestMemStore extends TestCase { assertNull(scanner.next()); } - private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected) { + private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected) + throws IOException { scanner.seek(KeyValue.createFirstOnRow(new byte[]{})); for (KeyValue kv : expected) { assertTrue(0 == @@ -209,7 +212,7 @@ public class TestMemStore extends TestCase { assertNull(scanner.peek()); } - public void testMemstoreConcurrentControl() { + public void testMemstoreConcurrentControl() throws IOException { final byte[] row = Bytes.toBytes(1); final byte[] f = Bytes.toBytes("family"); final byte[] q1 = Bytes.toBytes("q1"); @@ -250,7 +253,6 @@ public class TestMemStore extends TestCase { } private static class ReadOwnWritesTester extends Thread { - final int id; static final int NUM_TRIES = 1000; final byte[] row; @@ -269,7 +271,6 @@ public class TestMemStore extends TestCase { ReadWriteConsistencyControl rwcc, AtomicReference caughtException) { - this.id = id; this.rwcc = rwcc; this.memstore = memstore; this.caughtException = caughtException; @@ -284,7 +285,7 @@ public class TestMemStore extends TestCase { } } - private void internalRun() { + private void internalRun() throws IOException { for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) { ReadWriteConsistencyControl.WriteEntry w = rwcc.beginMemstoreInsert(); @@ -855,7 +856,7 @@ public class TestMemStore extends TestCase { } - static void doScan(MemStore ms, int iteration) { + static void doScan(MemStore ms, int iteration) throws IOException { long nanos = System.nanoTime(); KeyValueScanner s = ms.getScanners().get(0); s.seek(KeyValue.createFirstOnRow(new byte[]{})); @@ -868,7 +869,7 @@ public class TestMemStore extends TestCase { } - public static void main(String [] args) { + public static void main(String [] args) throws IOException { ReadWriteConsistencyControl rwcc = new ReadWriteConsistencyControl(); MemStore ms = new MemStore(); diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java index fd77329..2e4c7df 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -79,22 +79,23 @@ public class TestStoreFile extends HBaseTestCase { StoreFile.BloomType.NONE, false)); } + private void writeStoreFile(final HFile.Writer writer) throws IOException { + writeStoreFile(writer, Bytes.toBytes(getName()), Bytes.toBytes(getName())); + } /* * Writes HStoreKey and ImmutableBytes data to passed writer and * then closes it. * @param writer * @throws IOException */ - private void writeStoreFile(final HFile.Writer writer) + public static void writeStoreFile(final HFile.Writer writer, byte[] fam, byte[] qualifier) throws IOException { long now = System.currentTimeMillis(); - byte [] fam = Bytes.toBytes(getName()); - byte [] qf = Bytes.toBytes(getName()); try { for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) { for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) { byte[] b = new byte[] { (byte) d, (byte) e }; - writer.append(new KeyValue(b, fam, qf, now, b)); + writer.append(new KeyValue(b, fam, qualifier, now, b)); } } } finally {