diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java index 9fdd577..04ce0a6 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java @@ -36,6 +36,8 @@ public abstract class OperationWithAttributes extends Operation implements Attri // used for uniquely identifying an operation public static final String ID_ATRIBUTE = "_operation.attributes.id"; + // check only MemStore (Get, Scan, Increment, Append. Ignored by Delete and Put) + public static final String MEMSTORE_ONLY_ATTRIBUTE = "memstore_only"; @Override public OperationWithAttributes setAttribute(String name, byte[] value) { @@ -110,4 +112,25 @@ public abstract class OperationWithAttributes extends Operation implements Attri byte[] attr = getAttribute(ID_ATRIBUTE); return attr == null? null: Bytes.toString(attr); } + + /** + * This method allows you to set in-memory-only operation mode. + * Queries: Get and Scan as well as Increment and Append will + * work only on data in RAM (MemStore). + * If data is missing in RAM, the following will be returned: + * Empty Result for Get, Increment, Append and Scan + * + */ + public OperationWithAttributes setMemstoreOnly(boolean v){ + setAttribute(MEMSTORE_ONLY_ATTRIBUTE, Bytes.toBytes(v)); + return this; + } + /** + * Checks if we are in-memory-only mode + * @return true, if yes + */ + public boolean isMemstoreOnly(){ + byte[] attr = getAttribute(MEMSTORE_ONLY_ATTRIBUTE); + return attr != null && Bytes.toBoolean(attr) == true; + } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index c139296..abe1623 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -5262,7 +5262,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi this.filter = null; } this.comparator = region.getCellCompartor(); - + //HBASE-13890 + if(scan.isMemstoreOnly()){ + scan = new InternalScan(scan); + ((InternalScan) scan).checkOnlyMemStore(); + } + /** * By default, calls to next/nextRaw must enforce the batch limit. Thus, construct a default * scanner context that can be used to enforce the batch limit in the event that a @@ -6425,7 +6430,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } Scan scan = new Scan(get); - + //HBASE-13890 + scan.setMemstoreOnly(get.isMemstoreOnly()); RegionScanner scanner = null; try { scanner = getScanner(scan); @@ -6781,10 +6787,18 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi Collections.sort(family.getValue(), store.getComparator()); // Get previous values for all columns in this family Get get = new Get(row); - for (Cell cell : family.getValue()) { - get.addColumn(family.getKey(), CellUtil.cloneQualifier(cell)); + // HBASE-13890 + get.setMemstoreOnly(append.isMemstoreOnly()); + + for (Cell cell: family.getValue()) { + get.addColumn(family.getKey(), CellUtil.cloneQualifier(cell)); } + //get.setTimeRange(tr.getMin(), tr.getMax()); List results = get(get, false); + if(results.size() == 0 && get.isMemstoreOnly()){ + // Nothing was found - return empty result or null + return append.isReturnResults() ? Result.create(results) : null; + } // Iterate the input columns and update existing values if they were // found, otherwise add new column initialized to the append value @@ -7045,12 +7059,20 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi Collections.sort(family.getValue(), store.getComparator()); // Get previous values for all columns in this family Get get = new Get(row); + // HBASE-13890 + get.setMemstoreOnly(increment.isMemstoreOnly()); + for (Cell cell: family.getValue()) { get.addColumn(family.getKey(), CellUtil.cloneQualifier(cell)); } get.setTimeRange(tr.getMin(), tr.getMax()); List results = get(get, false); - + if(results.size() == 0 && get.isMemstoreOnly()){ + // memory store mode + // Nothing was found - return empty result or null + return increment.isReturnResults() ? Result.create(results) : null; + } + // Iterate the input columns and update existing values if they were // found, otherwise add new column initialized to the increment amount int idx = 0; diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreOnlyOperations.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreOnlyOperations.java new file mode 100644 index 0000000..8db63d7 --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreOnlyOperations.java @@ -0,0 +1,360 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestMemstoreOnlyOperations { + + @Test + public void testIncrement() throws Exception { + + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + FileSystem fs = FileSystem.get(conf); + byte[] table = Bytes.toBytes("table"); + byte[][] families = + new byte[][] { Bytes.toBytes("family1"), Bytes.toBytes("family2"), Bytes.toBytes("family3") }; + int numQualifiers = 3; + byte[][] qualifiers = new byte[numQualifiers][]; + for (int i = 0; i < numQualifiers; i++) + qualifiers[i] = Bytes.toBytes("qf" + i); + int numRows = 10; + byte[][] rows = new byte[numRows][]; + for (int i = 0; i < numRows; i++) + rows[i] = Bytes.toBytes("r" + i); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); + for (byte[] family : families) + htd.addFamily(new HColumnDescriptor(family)); + + HRegionInfo hri = new HRegionInfo(htd.getTableName(), null, null, false); + String testDir = htu.getDataTestDir() + "/TestMemstoreOnlyOperations/"; + Path path = new Path(testDir); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + Region region = HBaseTestingUtility.createRegionAndWAL(hri, path, conf, htd); + try { + Increment op = new Increment(rows[0]); + op.setDurability(Durability.SKIP_WAL); + + op.addColumn(families[0], qualifiers[0], 1); + op.addColumn(families[0], qualifiers[1], 2); + op.addColumn(families[0], qualifiers[2], 3); + + Result r = region.increment(op, HConstants.NO_NONCE, HConstants.NO_NONCE); + assertTrue(r.size() == 3); + Cell[] kvs = r.rawCells(); + for (int i = 0; i < kvs.length; i++) { + System.out.println(kvs[i].toString()); + assertTrue(CellUtil.matchingQualifier(kvs[i], qualifiers[i])); + assertEquals((i + 1), Bytes.toLong(CellUtil.cloneValue(kvs[i]))); + } + + // We have all in MemStore + Increment memop = new Increment(rows[0]); + memop.addColumn(families[0], qualifiers[0], 1); + memop.addColumn(families[0], qualifiers[1], 2); + memop.addColumn(families[0], qualifiers[2], 3); + memop.setMemstoreOnly(true); + + Result res = region.increment(memop, HConstants.NO_NONCE, HConstants.NO_NONCE); + + assertTrue(res.size() == r.size()); + kvs = res.rawCells(); + for (int i = 0; i < kvs.length; i++) { + System.out.println(kvs[i].toString()); + assertTrue(CellUtil.matchingQualifier(kvs[i], qualifiers[i])); + assertEquals((i + 1) * 2, Bytes.toLong(CellUtil.cloneValue(kvs[i]))); + } + + region.flush(true); + + res = region.increment(memop, HConstants.NO_NONCE, HConstants.NO_NONCE); + // After region flush - no more data in MemStore + assertTrue(res.size() == 0); + + } finally { + HBaseTestingUtility.closeRegionAndWAL(region); + } + HBaseTestingUtility.closeRegionAndWAL(region); + } + + @Test + public void testAppend() throws Exception { + + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + FileSystem fs = FileSystem.get(conf); + byte[] table = Bytes.toBytes("table"); + byte[][] families = + new byte[][] { Bytes.toBytes("family1"), Bytes.toBytes("family2"), Bytes.toBytes("family3") }; + int numQualifiers = 3; + byte[][] qualifiers = new byte[numQualifiers][]; + for (int i = 0; i < numQualifiers; i++) + qualifiers[i] = Bytes.toBytes("qf" + i); + int numRows = 10; + byte[][] rows = new byte[numRows][]; + for (int i = 0; i < numRows; i++) + rows[i] = Bytes.toBytes("r" + i); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); + for (byte[] family : families) + htd.addFamily(new HColumnDescriptor(family)); + + HRegionInfo hri = new HRegionInfo(htd.getTableName(), null, null, false); + String testDir = htu.getDataTestDir() + "/TestMemstoreOnlyOperations/"; + Path path = new Path(testDir); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + Region region = HBaseTestingUtility.createRegionAndWAL(hri, path, conf, htd); + try { + Append op = new Append(rows[0]); + op.setDurability(Durability.SKIP_WAL); + + op.add(families[0], qualifiers[0], "a".getBytes()); + op.add(families[0], qualifiers[1], "a".getBytes()); + op.add(families[0], qualifiers[2], "a".getBytes()); + + Result r = region.append(op, HConstants.NO_NONCE, HConstants.NO_NONCE); + assertTrue(r.size() == 3); + Cell[] kvs = r.rawCells(); + for (int i = 0; i < kvs.length; i++) { + System.out.println(kvs[i].toString()); + assertTrue(CellUtil.matchingQualifier(kvs[i], qualifiers[i])); + assertEquals("a", new String(kvs[i].getValue())); + } + + // We have all in MemStore + Append memop = new Append(rows[0]); + memop.add(families[0], qualifiers[0], "a".getBytes()); + memop.add(families[0], qualifiers[1], "a".getBytes()); + memop.add(families[0], qualifiers[2], "a".getBytes()); + memop.setMemstoreOnly(true); + + Result res = region.append(memop, HConstants.NO_NONCE, HConstants.NO_NONCE); + + assertTrue(res.size() == r.size()); + kvs = res.rawCells(); + for (int i = 0; i < kvs.length; i++) { + System.out.println(kvs[i].toString()); + assertTrue(CellUtil.matchingQualifier(kvs[i], qualifiers[i])); + assertEquals("aa", new String(kvs[i].getValue())); + } + + region.flush(true); + + res = region.append(memop, HConstants.NO_NONCE, HConstants.NO_NONCE); + // After region flush - no more data in MemStore + assertTrue(res.size() == 0); + + } finally { + HBaseTestingUtility.closeRegionAndWAL(region); + } + HBaseTestingUtility.closeRegionAndWAL(region); + } + + @Test + public void testGet() throws Exception { + + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + FileSystem fs = FileSystem.get(conf); + byte[] table = Bytes.toBytes("table"); + byte[][] families = + new byte[][] { Bytes.toBytes("family1"), Bytes.toBytes("family2"), Bytes.toBytes("family3") }; + int numQualifiers = 3; + byte[][] qualifiers = new byte[numQualifiers][]; + for (int i = 0; i < numQualifiers; i++) + qualifiers[i] = Bytes.toBytes("qf" + i); + int numRows = 10; + byte[][] rows = new byte[numRows][]; + for (int i = 0; i < numRows; i++) + rows[i] = Bytes.toBytes("r" + i); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); + for (byte[] family : families) + htd.addFamily(new HColumnDescriptor(family)); + + HRegionInfo hri = new HRegionInfo(htd.getTableName(), null, null, false); + String testDir = htu.getDataTestDir() + "/TestMemstoreOnlyOperations/"; + Path path = new Path(testDir); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + Region region = HBaseTestingUtility.createRegionAndWAL(hri, path, conf, htd); + try { + Put op = new Put(rows[0]); + op.setDurability(Durability.SKIP_WAL); + + op.addColumn(families[0], qualifiers[0], "a".getBytes()); + op.addColumn(families[0], qualifiers[1], "a".getBytes()); + op.addColumn(families[0], qualifiers[2], "a".getBytes()); + + region.put(op); + + Get get = new Get(rows[0]); + + get.addColumn(families[0], qualifiers[0]); + get.addColumn(families[0], qualifiers[1]); + get.addColumn(families[0], qualifiers[2]); + + Result r = region.get(get); + + assertTrue(r.size() == 3); + Cell[] kvs = r.rawCells(); + for (int i = 0; i < kvs.length; i++) { + System.out.println(kvs[i].toString()); + assertTrue(CellUtil.matchingQualifier(kvs[i], qualifiers[i])); + assertEquals("a", new String(kvs[i].getValue())); + } + + get.setMemstoreOnly(true); + Result res = region.get(get); + assertTrue(res.size() == r.size()); + kvs = res.rawCells(); + for (int i = 0; i < kvs.length; i++) { + System.out.println(kvs[i].toString()); + assertTrue(CellUtil.matchingQualifier(kvs[i], qualifiers[i])); + assertEquals("a", new String(kvs[i].getValue())); + } + region.flush(true); + res = region.get(get); + // After region flush - no more data in MemStore + assertTrue(res.size() == 0); + } finally { + HBaseTestingUtility.closeRegionAndWAL(region); + } + HBaseTestingUtility.closeRegionAndWAL(region); + } + + @Test + public void testScan() throws Exception { + + HBaseTestingUtility htu = new HBaseTestingUtility(); + Configuration conf = htu.getConfiguration(); + FileSystem fs = FileSystem.get(conf); + byte[] table = Bytes.toBytes("table"); + byte[][] families = + new byte[][] { Bytes.toBytes("family1"), Bytes.toBytes("family2"), Bytes.toBytes("family3") }; + int numQualifiers = 3; + byte[][] qualifiers = new byte[numQualifiers][]; + for (int i = 0; i < numQualifiers; i++) + qualifiers[i] = Bytes.toBytes("qf" + i); + int numRows = 10; + byte[][] rows = new byte[numRows][]; + for (int i = 0; i < numRows; i++) + rows[i] = Bytes.toBytes("r" + i); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); + for (byte[] family : families) + htd.addFamily(new HColumnDescriptor(family)); + + HRegionInfo hri = new HRegionInfo(htd.getTableName(), null, null, false); + String testDir = htu.getDataTestDir() + "/TestMemstoreOnlyOperations/"; + Path path = new Path(testDir); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + Region region = HBaseTestingUtility.createRegionAndWAL(hri, path, conf, htd); + try { + Put op = new Put(rows[0]); + op.setDurability(Durability.SKIP_WAL); + + op.addColumn(families[0], qualifiers[0], "a".getBytes()); + op.addColumn(families[0], qualifiers[1], "a".getBytes()); + op.addColumn(families[0], qualifiers[2], "a".getBytes()); + + region.put(op); + + Scan scan = new Scan(); + scan.setStartRow(rows[0]); + + scan.addColumn(families[0], qualifiers[0]); + scan.addColumn(families[0], qualifiers[1]); + scan.addColumn(families[0], qualifiers[2]); + + RegionScanner scanner = region.getScanner(scan); + + List cells = new ArrayList(); + scanner.next(cells); + assertTrue(cells.size() == 3); + for (int i = 0; i < cells.size(); i++) { + System.out.println(cells.get(i).toString()); + assertTrue(CellUtil.matchingQualifier(cells.get(i), qualifiers[i])); + assertEquals("a", new String(cells.get(i).getValue())); + } + scanner.close(); + scan.setMemstoreOnly(true); + + scanner = region.getScanner(scan); + cells = new ArrayList(); + scanner.next(cells); + assertTrue(cells.size() == 3); + for (int i = 0; i < cells.size(); i++) { + System.out.println(cells.get(i).toString()); + assertTrue(CellUtil.matchingQualifier(cells.get(i), qualifiers[i])); + assertEquals("a", new String(cells.get(i).getValue())); + } + scanner.close(); + region.flush(true); + cells = new ArrayList(); + + scanner = region.getScanner(scan); + scanner.next(cells); + // After region flush - no more data in MemStore + assertTrue(cells.size() == 0); + } finally { + HBaseTestingUtility.closeRegionAndWAL(region); + } + HBaseTestingUtility.closeRegionAndWAL(region); + } +}