Index: hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowMutations.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowMutations.java (revision 0) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowMutations.java (revision 0) @@ -0,0 +1,247 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import java.util.NavigableMap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.RowMutations; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestRowMutations extends HBaseTestCase { + + HRegion region = null; + + private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final String DIR = TEST_UTIL.getDataTestDir("TestAtomicOperation").toString(); + + static final byte[] tableName = Bytes.toBytes("TRM_testtable"); + static final byte [] row = Bytes.toBytes("rowA"); + static final byte[] cf1 = Bytes.toBytes("cf1"); + static final byte[] cf2 = Bytes.toBytes("cf2"); + static final byte[] qual1 = Bytes.toBytes("qual1"); + static final byte[] qual2 = Bytes.toBytes("qual2"); + static final byte[] value1 = Bytes.toBytes("value1"); + static final byte[] value2 = Bytes.toBytes("value2"); + + + /** + * Test {@link RowMutations} where we Delete a column family and then Put to same column family. + * @throws IOException + */ + public void testDeleteCFThenPutInSameCF() throws IOException { + initHRegion(tableName, getName(), HBaseConfiguration.create(), cf1); + + //delete entire column family + Delete delete = new Delete(row); + delete.deleteFamily(cf1); + + //Put to same column family + Put put = new Put(row); + put.add(cf1, qual1, value1); + put.add(cf1, qual2, value2); + + RowMutations rm = new RowMutations(row); + rm.add(delete); + rm.add(put); + region.mutateRow(rm); + + + Result result = region.get(new Get(row)); + NavigableMap familyMap = result.getFamilyMap(cf1); + assertNotNull(familyMap); + assertEquals(2, familyMap.size()); + assertEquals(value1, familyMap.get(qual1)); + assertEquals(value2, familyMap.get(qual2)); + } + + /** + * Test {@link RowMutations} where we Delete a row and then Put to same row. + * @throws IOException + */ + public void testDeleteRowThenPutSameRow() throws IOException { + initHRegion(tableName, getName(), HBaseConfiguration.create(), cf1); + + //delete row + Delete delete = new Delete(row); + + //Put to same row + Put put = new Put(row); + put.add(cf1, qual1, value1); + put.add(cf1, qual2, value2); + + RowMutations rm = new RowMutations(row); + rm.add(delete); + rm.add(put); + region.mutateRow(rm); + + + Result result = region.get(new Get(row)); + NavigableMap familyMap = result.getFamilyMap(cf1); + assertNotNull(familyMap); + assertEquals(2, familyMap.size()); + assertEquals(value1, familyMap.get(qual1)); + assertEquals(value2, familyMap.get(qual2)); + } + + /** + * Test {@link RowMutations} where we Put to a column and then Delete same column. + * @throws IOException + */ + public void testPutToColumnThenDeleteSameColumn() throws IOException { + initHRegion(tableName, getName(), HBaseConfiguration.create(), cf1); + + //Put to some columns + Put put = new Put(row); + put.add(cf1, qual1, value1); + put.add(cf1, qual2, value2); + + //delete same columns + Delete delete = new Delete(row); + delete.deleteColumn(cf1, qual1); + delete.deleteColumn(cf1, qual2); + + RowMutations rm = new RowMutations(row); + rm.add(put); + rm.add(delete); + + region.mutateRow(rm); + + + Result result = region.get(new Get(row)); + NavigableMap familyMap = result.getFamilyMap(cf1); + assertNull(familyMap); + } + + /** + * Test {@link RowMutations} where we Delete a column family and then Put to different column family. Passes currently. + * @throws IOException + */ + public void testDeleteCFThenPutInDiffCF() throws IOException { + initHRegion(tableName, getName(), HBaseConfiguration.create(), cf1, cf2); + + //delete some column family + Delete delete = new Delete(row); + delete.deleteFamily(cf1); + + //Put to different column family + Put put = new Put(row); + put.add(cf2, qual1, value1); + put.add(cf2, qual2, value2); + + RowMutations rm = new RowMutations(row); + rm.add(delete); + rm.add(put); + region.mutateRow(rm); + + + Result result = region.get(new Get(row)); + NavigableMap familyMap = result.getFamilyMap(cf2); + assertNotNull(familyMap); + assertEquals(2, familyMap.size()); + assertEquals(value1, familyMap.get(qual1)); + assertEquals(value2, familyMap.get(qual2)); + + NavigableMap familyMap1 = result.getFamilyMap(cf1); + assertEquals(0, familyMap1.size()); //TODO: why is familyMap1 empty instead of null? + + } + + /** + * Test {@link RowMutations} where we Delete a column and then Put to different column. Passes currently. + * @throws IOException + */ + public void testDeleteColumnThenPutDiffColumn() throws IOException { + initHRegion(tableName, getName(), HBaseConfiguration.create(), cf1, cf2); + + //delete columns + Delete delete = new Delete(row); + delete.deleteColumn(cf1, qual1); + + //Put to different column, one in same column family and one in another column family + Put put = new Put(row); + put.add(cf1, qual2, value2); + put.add(cf2, qual2, value2); + + RowMutations rm = new RowMutations(row); + rm.add(delete); + rm.add(put); + region.mutateRow(rm); + + + Result result = region.get(new Get(row)); + NavigableMap familyMap = result.getFamilyMap(cf1); + assertNotNull(familyMap); + assertEquals(1, familyMap.size()); + assertEquals(value2, familyMap.get(qual2)); + + NavigableMap familyMap2 = result.getFamilyMap(cf2); + assertNotNull(familyMap2); + assertEquals(1, familyMap2.size()); + assertEquals(value2, familyMap2.get(qual2)); + + } + + + + private void initHRegion (byte [] tableName, String callingMethod, + Configuration conf, byte [] ... families) + throws IOException{ + HTableDescriptor htd = new HTableDescriptor(tableName); + for(byte [] family : families) { + htd.addFamily(new HColumnDescriptor(family)); + } + HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false); + Path path = new Path(DIR + callingMethod); + if (fs.exists(path)) { + if (!fs.delete(path, true)) { + throw new IOException("Failed delete of " + path); + } + } + region = HRegion.createHRegion(info, path, conf, htd); + } + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + EnvironmentEdgeManagerTestHelper.reset(); + } +} Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiRowMutationProcessor.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiRowMutationProcessor.java (revision 1486362) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiRowMutationProcessor.java (working copy) @@ -21,15 +21,17 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; -import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException; import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProcessorProtos.MultiRowMutationProcessorRequest; import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProcessorProtos.MultiRowMutationProcessorResponse; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -69,7 +71,16 @@ HRegion region, List mutationKvs, WALEdit walEdit) throws IOException { + byte[] bytePrev = Bytes.toBytes(now-1); byte[] byteNow = Bytes.toBytes(now); + Set familiesInPuts = new TreeSet(Bytes.BYTES_RAWCOMPARATOR); + // find column families referenced by Puts + for (Mutation m : mutations) { + if (m instanceof Put) { + Map> familyMap = m.getFamilyMap(); + familiesInPuts.addAll(familyMap.keySet()); + } + } // Check mutations and apply edits to a single WALEdit for (Mutation m : mutations) { if (m instanceof Put) { @@ -80,22 +91,32 @@ } else if (m instanceof Delete) { Delete d = (Delete) m; region.prepareDelete(d); - region.prepareDeleteTimestamps(d.getFamilyMap(), byteNow); } else { throw new DoNotRetryIOException( "Action must be Put or Delete. But was: " + m.getClass().getName()); } - for (List cells: m.getFamilyMap().values()) { + byte[] byteForTime = byteNow; + for (Map.Entry> entry: m.getFamilyMap().entrySet()) { + List cells = entry.getValue(); boolean writeToWAL = m.getDurability() != Durability.SKIP_WAL; for (Cell cell : cells) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + // if a column family referenced in one of the Puts is deleted, we use earlier timestamp + if (kv.getType() == KeyValue.Type.DeleteFamily.getCode() && + familiesInPuts.contains(entry.getKey())) { + byteForTime = bytePrev; + } mutationKvs.add(kv); if (writeToWAL) { walEdit.add(kv); } } } + if (m instanceof Delete) { + Delete d = (Delete) m; + region.prepareDeleteTimestamps(d.getFamilyMap(), byteForTime); + } } }