diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java index a1da601..b1f8a78 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java @@ -179,7 +179,7 @@ public void cleanUpCluster() throws Exception { numBackReferencesPerRow = conf.getInt(NUM_BACKREFS_KEY, NUM_BACKREFS_DEFAULT); table = new HTable(conf, tableName); table.setWriteBufferSize(4*1024*1024); - table.setAutoFlush(false, true); + table.setAutoFlush(true, true); String taskId = conf.get("mapreduce.task.attempt.id"); Matcher matcher = Pattern.compile(".+_m_(\\d+_\\d+)").matcher(taskId); diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java index 7b59909..5bfda0c 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java @@ -22,7 +22,9 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.security.PrivilegedExceptionAction; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.commons.cli.CommandLine; import org.apache.hadoop.conf.Configuration; @@ -34,6 +36,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.IntegrationTestingUtility; import org.apache.hadoop.hbase.IntegrationTests; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -94,16 +97,29 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT private static final String NUM_TO_WRITE_KEY = "loadmapper.num_to_write"; private static final long NUM_TO_WRITE_DEFAULT = 100 * 1000; private static final int SCANNER_CACHING = 500; - + private static Map expToInt = new HashMap(); + static { + expToInt.put(CONFIDENTIAL + AND + TOPSECRET + AND + PRIVATE, 0); + expToInt.put(CONFIDENTIAL + OR + TOPSECRET, 1); + expToInt.put(PUBLIC, 2); + expToInt.put('(' + SECRET + OR + PRIVATE + ')' + AND + NOT + CONFIDENTIAL, 3); + } private long numRowsLoadedWithExp1, numRowsLoadedWithExp2, numRowsLoadWithExp3, numRowsLoadWithExp4; private long numRowsReadWithExp1, numRowsReadWithExp2, numRowsReadWithExp3, numRowsReadWithExp4; + + private long numRowsDeletedWithExp1, numRowsDeletedWithExp2, numRowsDeletedWithExp3, + numRowsDeletedWithExp4; private static User USER1, USER2; private enum Counters { ROWS_VIS_EXP_1, ROWS_VIS_EXP_2, ROWS_VIS_EXP_3, ROWS_VIS_EXP_4; } + + private enum DelCounters { + ROWS_VIS_EXP_DEL_1, ROWS_VIS_EXP_DEL_2, ROWS_VIS_EXP_DEL_3, ROWS_VIS_EXP_DEL_4; + } @Override public void setUpCluster() throws Exception { @@ -132,7 +148,8 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT } public static class LoadWithCellVisibilityMapper extends LoadMapper { - private Counter rowsExp1, rowsExp2, rowsExp3, rowsexp4; + private Counter rowsExp1, rowsExp2, rowsExp3, rowsExp4; + private Counter rowsDelExp1, rowsDelExp2, rowsDelExp3, rowsDelExp4; @Override public void setup(Context context) throws IOException { @@ -140,7 +157,12 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT rowsExp1 = context.getCounter(Counters.ROWS_VIS_EXP_1); rowsExp2 = context.getCounter(Counters.ROWS_VIS_EXP_2); rowsExp3 = context.getCounter(Counters.ROWS_VIS_EXP_3); - rowsexp4 = context.getCounter(Counters.ROWS_VIS_EXP_4); + rowsExp4 = context.getCounter(Counters.ROWS_VIS_EXP_4); + + rowsDelExp1 = context.getCounter(DelCounters.ROWS_VIS_EXP_DEL_1); + rowsDelExp2 = context.getCounter(DelCounters.ROWS_VIS_EXP_DEL_2); + rowsDelExp3 = context.getCounter(DelCounters.ROWS_VIS_EXP_DEL_3); + rowsDelExp4 = context.getCounter(DelCounters.ROWS_VIS_EXP_DEL_4); } @Override @@ -148,16 +170,32 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT InterruptedException { String suffix = "/" + shortTaskId; int BLOCK_SIZE = (int) (recordsToWrite / 100); + String temp = null; for (long i = 0; i < recordsToWrite;) { for (long idx = 0; idx < BLOCK_SIZE && i < recordsToWrite; idx++, i++) { int expIdx = rand.nextInt(BLOCK_SIZE) % VISIBILITY_EXPS_COUNT; String exp = VISIBILITY_EXPS[expIdx]; - byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp)); - Put p = new Put(row); - p.add(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); - p.setCellVisibility(new CellVisibility(exp)); - getCounter(expIdx).increment(1); - table.put(p); + // Every 10th element before the 100th element is stored and applied as delete + // The same expression id is also incremented + if (i != 0 && i % 100 == 0) { + byte[] row = Bytes.add(Bytes.toBytes(i - 10), Bytes.toBytes(suffix), + Bytes.toBytes(temp)); + Delete d = new Delete(row); + d.deleteColumn(TEST_FAMILY, TEST_QUALIFIER); + d.setCellVisibility(new CellVisibility(temp)); + getCounter(expToInt.get(temp), true).increment(1); + table.delete(d); + } else { + if (i != 0 && (i + 10) % 100 == 0) { + temp = exp; + } + byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp)); + Put p = new Put(row); + p.add(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); + p.setCellVisibility(new CellVisibility(exp)); + getCounter(expIdx, false).increment(1); + table.put(p); + } if (i % 100 == 0) { context.setStatus("Written " + i + "/" + recordsToWrite + " records"); @@ -170,16 +208,32 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT } } - private Counter getCounter(int idx) { + private Counter getCounter(int idx, boolean isDel) { switch (idx) { case 0: - return rowsExp1; + if (!isDel) { + return rowsExp1; + } else { + return rowsDelExp1; + } case 1: - return rowsExp2; + if (!isDel) { + return rowsExp2; + } else { + return rowsDelExp2; + } case 2: - return rowsExp3; + if (!isDel) { + return rowsExp3; + } else { + return rowsDelExp3; + } case 3: - return rowsexp4; + if (!isDel) { + return rowsExp4; + } else { + return rowsDelExp4; + } default: return null; } @@ -227,14 +281,27 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT this.numRowsLoadedWithExp2 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_2).getValue(); this.numRowsLoadWithExp3 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_3).getValue(); this.numRowsLoadWithExp4 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_4).getValue(); + + this.numRowsDeletedWithExp1 = job.getCounters().findCounter(DelCounters.ROWS_VIS_EXP_DEL_1).getValue(); + this.numRowsDeletedWithExp2 = job.getCounters().findCounter(DelCounters.ROWS_VIS_EXP_DEL_2).getValue(); + this.numRowsDeletedWithExp3 = job.getCounters().findCounter(DelCounters.ROWS_VIS_EXP_DEL_3).getValue(); + this.numRowsDeletedWithExp4 = job.getCounters().findCounter(DelCounters.ROWS_VIS_EXP_DEL_4).getValue(); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[0] + " : " + this.numRowsLoadedWithExp1); + System.out.println("Rows deleted with cell visibility " + VISIBILITY_EXPS[0] + " : " + + this.numRowsDeletedWithExp1); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[1] + " : " + this.numRowsLoadedWithExp2); + System.out.println("Rows deleted with cell visibility " + VISIBILITY_EXPS[1] + " : " + + this.numRowsDeletedWithExp2); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[2] + " : " + this.numRowsLoadWithExp3); + System.out.println("Rows deleted with cell visibility " + VISIBILITY_EXPS[2] + " : " + + this.numRowsDeletedWithExp3); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[3] + " : " + this.numRowsLoadWithExp4); + System.out.println("Rows deleted with cell visibility " + VISIBILITY_EXPS[3] + " : " + + this.numRowsDeletedWithExp4); return job; } @@ -256,8 +323,8 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT this.numRowsReadWithExp2 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_2).getValue(); this.numRowsReadWithExp3 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_3).getValue(); this.numRowsReadWithExp4 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_4).getValue(); - assertEquals(this.numRowsLoadedWithExp1, this.numRowsReadWithExp1); - assertEquals(this.numRowsLoadedWithExp2, this.numRowsReadWithExp2); + assertEquals(this.numRowsLoadedWithExp1 - this.numRowsDeletedWithExp1, this.numRowsReadWithExp1); + assertEquals(this.numRowsLoadedWithExp2 - this.numRowsDeletedWithExp2, this.numRowsReadWithExp2); assertEquals(0, this.numRowsReadWithExp3); assertEquals(0, this.numRowsReadWithExp4); @@ -277,7 +344,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT assertEquals(0, this.numRowsReadWithExp1); assertEquals(0, this.numRowsReadWithExp2); assertEquals(0, this.numRowsReadWithExp3); - assertEquals(this.numRowsLoadWithExp4, this.numRowsReadWithExp4); + assertEquals(this.numRowsLoadWithExp4 - this.numRowsDeletedWithExp4, this.numRowsReadWithExp4); // Normal user only having PUBLIC label auth and can view only those cells. System.out.println(String.format("Verifying for auths %s, %s", PRIVATE, PUBLIC)); @@ -294,7 +361,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT this.numRowsReadWithExp4 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_4).getValue(); assertEquals(0, this.numRowsReadWithExp1); assertEquals(0, this.numRowsReadWithExp2); - assertEquals(this.numRowsLoadWithExp3, this.numRowsReadWithExp3); + assertEquals(this.numRowsLoadWithExp3 - this.numRowsDeletedWithExp3, this.numRowsReadWithExp3); assertEquals(0, this.numRowsReadWithExp4); }