Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java (revision ) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java (revision ) @@ -0,0 +1,98 @@ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +/** + * Test case to test that HRS doesn't crash with OOME when + * the row is too big. + */ +@Category(MediumTests.class) +public class TestRowTooBig { + private final static HBaseTestingUtility HTU = HBaseTestingUtility.createLocalHTU(); + private static final HTableDescriptor TEST_HTD = + new HTableDescriptor(TableName.valueOf(TestRowTooBig.class.getSimpleName())); + + @BeforeClass + public static void before() throws Exception { + HTU.startMiniCluster(); + } + + @AfterClass + public static void after() throws Exception { + HTU.shutdownMiniCluster(); + } + + /** + * OOME happens before we actually get to reading results, but + * during seeking the scanners in HFiles. Is that meaningful usecase? + * @throws IOException + */ + @Test(expected = RowTooBigException.class) + public void testFewHugeCells() throws IOException { + byte[] row1 = Bytes.toBytes("row1"); + byte[] fam1 = Bytes.toBytes("fam1"); + + HTableDescriptor htd = TEST_HTD; + HColumnDescriptor hcd = new HColumnDescriptor(fam1); + htd.addFamily(hcd); + + final HRegionInfo hri = + new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW, + HConstants.EMPTY_END_ROW); + HRegion region = HTU.createLocalHRegion(hri, htd); + + // Add to memstore + for (int i = 0; i < 5 ; i++) { + Put put = new Put(row1); + + // putting 5 MB cells + put.add(fam1, Bytes.toBytes("col_" + i ), new byte[5 * 1024 * 1024]); + region.put(put); + region.flushcache(); + // but not compacting! + region.flushcache(); + } + + Get get = new Get(row1); + Result res = region.get(get); + } + + @Test(expected = RowTooBigException.class) + public void testMillionSmallColumns() throws IOException { + byte[] row1 = Bytes.toBytes("row1"); + byte[] fam1 = Bytes.toBytes("fam1"); + + HTableDescriptor htd = TEST_HTD; + HColumnDescriptor hcd = new HColumnDescriptor(fam1); + htd.addFamily(hcd); + + final HRegionInfo hri = + new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW, + HConstants.EMPTY_END_ROW); + HRegion region = HTU.createLocalHRegion(hri, htd); + + // Add to memstore + for (int i = 0; i < 100; i++) { + Put put = new Put(row1); + for (int j = 0; j < 10000; j++) { + put.add(fam1, Bytes.toBytes("col_" + i + "_" + j), new byte[10]); + } + region.put(put); + region.flushcache(); + } + region.compactStores(true); + + Get get = new Get(row1); + Result res = region.get(get); + } +} Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (revision 1cf2c530779316f1cd3cd41f9d3834365e02e423) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (revision ) @@ -313,9 +313,19 @@ } } else { if (!isParallelSeek) { + long maxRowSize = 10 * 1024 * 1024; + long totalScannersSought = 0; for (KeyValueScanner scanner : scanners) { + if (totalScannersSought >= maxRowSize ) { + throw new RowTooBigException("Max row size allowed: " + maxRowSize + ", but row is bigger than that"); + } scanner.seek(seekKey); + Cell c = scanner.peek(); + if (c != null ) { + totalScannersSought += c.getRowLength() + c.getFamilyLength() + + c.getQualifierLength() + c.getValueLength(); - } + } + } } else { parallelSeek(scanners, seekKey); } @@ -461,6 +471,10 @@ store != null ? store.getComparator() : null; int count = 0; + + long maxRowSize = 10 * 1024 * 1024; + long totalReadSize = 0; + LOOP: while((kv = this.heap.peek()) != null) { if (prevKV != kv) ++kvsScanned; // Do object compare - we set prevKV from the same heap. checkScanOrder(prevKV, kv, comparator); @@ -494,6 +508,12 @@ if (this.countPerRow > storeOffset) { outResult.add(kv); count++; + totalReadSize += kv.getRowLength() + kv.getFamilyLength() + + kv.getQualifierLength() + kv.getValueLength(); + if (totalReadSize > maxRowSize) { + throw new RowTooBigException("Max size of row is :" + maxRowSize + + ", but the row is bigger than that."); + } } if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW) { Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RowTooBigException.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RowTooBigException.java (revision ) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RowTooBigException.java (revision ) @@ -0,0 +1,14 @@ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.RegionException; + +/** + * Get or Scans throw this exception when row size + * appears to exceed max configured row size. + */ +public class RowTooBigException extends RegionException { + + public RowTooBigException(String message) { + super(message); + } +}