diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java index a1da601..b1f8a78 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java @@ -179,7 +179,7 @@ public void cleanUpCluster() throws Exception { numBackReferencesPerRow = conf.getInt(NUM_BACKREFS_KEY, NUM_BACKREFS_DEFAULT); table = new HTable(conf, tableName); table.setWriteBufferSize(4*1024*1024); - table.setAutoFlush(false, true); + table.setAutoFlush(true, true); String taskId = conf.get("mapreduce.task.attempt.id"); Matcher matcher = Pattern.compile(".+_m_(\\d+_\\d+)").matcher(taskId); diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java index 7b59909..4ec888a 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.IntegrationTestingUtility; import org.apache.hadoop.hbase.IntegrationTests; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -99,10 +100,14 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT numRowsLoadWithExp4; private long numRowsReadWithExp1, numRowsReadWithExp2, numRowsReadWithExp3, numRowsReadWithExp4; + private long numRowsDeletedWithExp1, numRowsDeletedWithExp2, numRowsDeletedWithExp3, + numRowsDeletedWithExp4; + private static User USER1, USER2; private enum Counters { - ROWS_VIS_EXP_1, ROWS_VIS_EXP_2, ROWS_VIS_EXP_3, ROWS_VIS_EXP_4; + ROWS_VIS_EXP_1, ROWS_VIS_EXP_2, ROWS_VIS_EXP_3, ROWS_VIS_EXP_4, + ROWS_VIS_EXP_DEL_1, ROWS_VIS_EXP_DEL_2, ROWS_VIS_EXP_DEL_3, ROWS_VIS_EXP_DEL_4; } @Override @@ -132,7 +137,8 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT } public static class LoadWithCellVisibilityMapper extends LoadMapper { - private Counter rowsExp1, rowsExp2, rowsExp3, rowsexp4; + private Counter rowsExp1, rowsExp2, rowsExp3, rowsExp4; + private Counter rowsDelExp1, rowsDelExp2, rowsDelExp3, rowsDelExp4; @Override public void setup(Context context) throws IOException { @@ -140,7 +146,12 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT rowsExp1 = context.getCounter(Counters.ROWS_VIS_EXP_1); rowsExp2 = context.getCounter(Counters.ROWS_VIS_EXP_2); rowsExp3 = context.getCounter(Counters.ROWS_VIS_EXP_3); - rowsexp4 = context.getCounter(Counters.ROWS_VIS_EXP_4); + rowsExp4 = context.getCounter(Counters.ROWS_VIS_EXP_4); + + rowsDelExp1 = context.getCounter(Counters.ROWS_VIS_EXP_DEL_1); + rowsDelExp2 = context.getCounter(Counters.ROWS_VIS_EXP_DEL_2); + rowsDelExp3 = context.getCounter(Counters.ROWS_VIS_EXP_DEL_3); + rowsDelExp4 = context.getCounter(Counters.ROWS_VIS_EXP_DEL_4); } @Override @@ -148,16 +159,32 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT InterruptedException { String suffix = "/" + shortTaskId; int BLOCK_SIZE = (int) (recordsToWrite / 100); + int tempIdx = -1;; for (long i = 0; i < recordsToWrite;) { for (long idx = 0; idx < BLOCK_SIZE && i < recordsToWrite; idx++, i++) { int expIdx = rand.nextInt(BLOCK_SIZE) % VISIBILITY_EXPS_COUNT; String exp = VISIBILITY_EXPS[expIdx]; - byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp)); - Put p = new Put(row); - p.add(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); - p.setCellVisibility(new CellVisibility(exp)); - getCounter(expIdx).increment(1); - table.put(p); + // Every 10th element before the 100th element is stored and applied as delete + // The same expression id is also incremented + if (i != 0 && i % 100 == 0) { + byte[] row = Bytes.add(Bytes.toBytes(i - 10), Bytes.toBytes(suffix), + Bytes.toBytes(VISIBILITY_EXPS[tempIdx])); + Delete d = new Delete(row); + d.deleteColumn(TEST_FAMILY, TEST_QUALIFIER); + d.setCellVisibility(new CellVisibility(VISIBILITY_EXPS[tempIdx])); + getCounter(tempIdx, true).increment(1); + table.delete(d); + } else { + if (i != 0 && (i + 10) % 100 == 0) { + tempIdx = expIdx; + } + byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp)); + Put p = new Put(row); + p.add(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); + p.setCellVisibility(new CellVisibility(exp)); + getCounter(expIdx, false).increment(1); + table.put(p); + } if (i % 100 == 0) { context.setStatus("Written " + i + "/" + recordsToWrite + " records"); @@ -170,16 +197,32 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT } } - private Counter getCounter(int idx) { + private Counter getCounter(int idx, boolean isDel) { switch (idx) { case 0: - return rowsExp1; + if (!isDel) { + return rowsExp1; + } else { + return rowsDelExp1; + } case 1: - return rowsExp2; + if (!isDel) { + return rowsExp2; + } else { + return rowsDelExp2; + } case 2: - return rowsExp3; + if (!isDel) { + return rowsExp3; + } else { + return rowsDelExp3; + } case 3: - return rowsexp4; + if (!isDel) { + return rowsExp4; + } else { + return rowsDelExp4; + } default: return null; } @@ -227,14 +270,31 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT this.numRowsLoadedWithExp2 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_2).getValue(); this.numRowsLoadWithExp3 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_3).getValue(); this.numRowsLoadWithExp4 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_4).getValue(); + + this.numRowsDeletedWithExp1 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_DEL_1) + .getValue(); + this.numRowsDeletedWithExp2 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_DEL_2) + .getValue(); + this.numRowsDeletedWithExp3 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_DEL_3) + .getValue(); + this.numRowsDeletedWithExp4 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_DEL_4) + .getValue(); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[0] + " : " + this.numRowsLoadedWithExp1); + System.out.println("Rows deleted with cell visibility " + VISIBILITY_EXPS[0] + " : " + + this.numRowsDeletedWithExp1); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[1] + " : " + this.numRowsLoadedWithExp2); + System.out.println("Rows deleted with cell visibility " + VISIBILITY_EXPS[1] + " : " + + this.numRowsDeletedWithExp2); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[2] + " : " + this.numRowsLoadWithExp3); + System.out.println("Rows deleted with cell visibility " + VISIBILITY_EXPS[2] + " : " + + this.numRowsDeletedWithExp3); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[3] + " : " + this.numRowsLoadWithExp4); + System.out.println("Rows deleted with cell visibility " + VISIBILITY_EXPS[3] + " : " + + this.numRowsDeletedWithExp4); return job; } @@ -256,8 +316,8 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT this.numRowsReadWithExp2 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_2).getValue(); this.numRowsReadWithExp3 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_3).getValue(); this.numRowsReadWithExp4 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_4).getValue(); - assertEquals(this.numRowsLoadedWithExp1, this.numRowsReadWithExp1); - assertEquals(this.numRowsLoadedWithExp2, this.numRowsReadWithExp2); + assertEquals(this.numRowsLoadedWithExp1 - this.numRowsDeletedWithExp1, this.numRowsReadWithExp1); + assertEquals(this.numRowsLoadedWithExp2 - this.numRowsDeletedWithExp2, this.numRowsReadWithExp2); assertEquals(0, this.numRowsReadWithExp3); assertEquals(0, this.numRowsReadWithExp4); @@ -277,7 +337,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT assertEquals(0, this.numRowsReadWithExp1); assertEquals(0, this.numRowsReadWithExp2); assertEquals(0, this.numRowsReadWithExp3); - assertEquals(this.numRowsLoadWithExp4, this.numRowsReadWithExp4); + assertEquals(this.numRowsLoadWithExp4 - this.numRowsDeletedWithExp4, this.numRowsReadWithExp4); // Normal user only having PUBLIC label auth and can view only those cells. System.out.println(String.format("Verifying for auths %s, %s", PRIVATE, PUBLIC)); @@ -294,7 +354,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT this.numRowsReadWithExp4 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_4).getValue(); assertEquals(0, this.numRowsReadWithExp1); assertEquals(0, this.numRowsReadWithExp2); - assertEquals(this.numRowsLoadWithExp3, this.numRowsReadWithExp3); + assertEquals(this.numRowsLoadWithExp3 - this.numRowsDeletedWithExp3, this.numRowsReadWithExp3); assertEquals(0, this.numRowsReadWithExp4); }