From 7cdff1fe423e2f653575b61fe18e5ca5600814fd Mon Sep 17 00:00:00 2001 From: Jonathan Lawlor Date: Tue, 31 Mar 2015 16:55:24 -0700 Subject: [PATCH 1/2] Issue with small scanners --- .../hbase/TestSmallScannerFromClientSide.java | 176 +++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/TestSmallScannerFromClientSide.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSmallScannerFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSmallScannerFromClientSide.java new file mode 100644 index 0000000..5da0e1b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSmallScannerFromClientSide.java @@ -0,0 +1,176 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestSmallScannerFromClientSide { + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private static Table TABLE = null; + + /** + * Table configuration + */ + private static TableName TABLE_NAME = TableName.valueOf("testTable"); + + private static int NUM_ROWS = 5; + private static byte[] ROW = Bytes.toBytes("testRow"); + private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS); + + private static int NUM_FAMILIES = 10; + private static byte[] FAMILY = Bytes.toBytes("testFamily"); + private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES); + + private static int NUM_QUALIFIERS = 10; + private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS); + + private static int VALUE_SIZE = 1024; + private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + // Configure to be some value less than Long.MAX_VALUE and greater than the size of a row. + // 2GB does the job + conf.setLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, 2 * 1024 * 1024); + TEST_UTIL.startMiniCluster(3); + TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE); + } + + static Table createTestTable(TableName name, byte[][] rows, byte[][] families, + byte[][] qualifiers, byte[] cellValue) throws IOException { + Table ht = TEST_UTIL.createTable(name, families); + List puts = createPuts(rows, families, qualifiers, cellValue); + ht.put(puts); + + return ht; + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Make puts to put the input value into each combination of row, family, and qualifier + * @param rows + * @param families + * @param qualifiers + * @param value + * @return + * @throws IOException + */ + static ArrayList createPuts(byte[][] rows, byte[][] families, byte[][] qualifiers, + byte[] value) throws IOException { + Put put; + ArrayList puts = new ArrayList<>(); + + for (int row = 0; row < rows.length; row++) { + put = new Put(rows[row]); + for (int fam = 0; fam < families.length; fam++) { + for (int qual = 0; qual < qualifiers.length; qual++) { + KeyValue kv = new KeyValue(rows[row], families[fam], qualifiers[qual], qual, value); + put.add(kv); + } + } + puts.add(put); + } + + return puts; + } + + /** + * Make key values to represent each possible combination of family and qualifier in the specified + * row. + * @param row + * @param families + * @param qualifiers + * @param value + * @return + */ + static ArrayList createKeyValuesForRow(byte[] row, byte[][] families, byte[][] qualifiers, + byte[] value) { + ArrayList outList = new ArrayList<>(); + for (int fam = 0; fam < families.length; fam++) { + for (int qual = 0; qual < qualifiers.length; qual++) { + outList.add(new KeyValue(row, families[fam], qualifiers[qual], qual, value)); + } + } + return outList; + } + + @Test + public void testSmallScanWithRestrictiveMaxResultSize() throws Exception { + Scan scan = new Scan(); + scan.setSmall(true); + // Configure max result size to be smaller than the size of a single row. + scan.setMaxResultSize(1); + + ResultScanner scanner = TABLE.getScanner(scan); + int rowCount = 0; + while (scanner.next() != null) { + rowCount++; + } + + assertTrue("Expect rows: " + NUM_ROWS + " got: " + rowCount, rowCount == NUM_ROWS); + + scanner.close(); + } + + @Test + public void testSmallScanWithMaximumCachingValue() throws Exception { + Scan scan = new Scan(); + scan.setSmall(true); + scan.setCaching(Integer.MAX_VALUE); + int numCols = NUM_FAMILIES * NUM_QUALIFIERS; + + // Configure max result size so that the size limit is reached before the entire table is + // exhausted. We want there to be at least 2 scan RPCs + scan.setMaxResultSize((NUM_ROWS - 1) * numCols * VALUE_SIZE); + + ResultScanner scanner = TABLE.getScanner(scan); + int rowCount = 0; + while (scanner.next() != null) { + rowCount++; + } + + assertTrue("Expect rows: " + NUM_ROWS + " got: " + rowCount, rowCount == NUM_ROWS); + + scanner.close(); + } +} \ No newline at end of file -- 1.9.3 (Apple Git-50)