From 3a1ad76887250464758ef057eba2d5eb521c3c28 Mon Sep 17 00:00:00 2001 From: Umesh Agashe Date: Sat, 9 Sep 2017 02:10:00 -0700 Subject: [PATCH] HBASE-13271 Added test for batch operations with validation errors. Updated Javadoc for batch methods. Javadoc for following methods are updated: * Table.put(List puts) * Table.delete(List deletes) Added @apiNote for delete regarding input list will not be modied in version 3.0.0 --- .../org/apache/hadoop/hbase/client/HTable.java | 1 + .../java/org/apache/hadoop/hbase/client/Table.java | 22 +++- .../hadoop/hbase/client/TestFromClientSide.java | 132 +++++++++++++++++---- .../hadoop/hbase/master/TestMasterFailover.java | 4 - 4 files changed, 129 insertions(+), 30 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java index 0ca26f01e50647d92e9a3e29c8cdbb9d33e94860..8b3ca963dabb41aac7a0488624d8d90d49cfad63 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -552,6 +552,7 @@ public class HTable implements Table { } catch (InterruptedException e) { throw (InterruptedIOException)new InterruptedIOException().initCause(e); } finally { + // TODO: to be consistent with batch put(), do not modify input list // mutate list so that it is empty for complete success, or contains only failed records // results are returned in the same order as the requests in list walk the list backwards, // so we can remove from list without impacting the indexes of earlier members diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java index 66d4616d56ccbf3c3054a1911dc6e72dd9914fa4..620485e33cda94e034b9132bbe578ac96b7220ef 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java @@ -209,7 +209,12 @@ public interface Table extends Closeable { /** * Puts some data in the table, in batch. *

- * This can be used for group commit, or for submitting user defined batches. + * This can be used for group commit, or for submitting user defined batches. If a column is not + * specified for any of the mutations in the input list, {@link IllegalArgumentException} will be + * thrown and no mutations will be applied. If there are any failures even after retries, + * {@link RetriesExhaustedWithDetailsException}, containing list of failed mutations and + * respective remote exceptions will be thrown. The ordering of mutations and exceptions in the + * encapsulating exception corresponds to the order of the input list of Put requests. * @param puts The list of mutations to apply. * @throws IOException if a remote or network exception occurs. * @since 0.20.0 @@ -290,14 +295,21 @@ public interface Table extends Closeable { /** * Deletes the specified cells/rows in bulk. - * @param deletes List of things to delete. List gets modified by this - * method (in particular it gets re-ordered, so the order in which the elements - * are inserted in the list gives no guarantee as to the order in which the - * {@link Delete}s are executed). + *

+ * If row specified doesn't exist, {@link Delete} will be successfully applied and no exception + * is thrown. If there are any failures even after retries, + * {@link RetriesExhaustedWithDetailsException}, containing list of failed {@link Delete}s and + * respective remote exceptions will be thrown. + * + * @param deletes List of things to delete. The input list gets modified by this + * method. All successfully applied {@link Delete}s in the list are removed. (in particular it + * gets re-ordered, so the order in which the elements are inserted in the list gives no + * guarantee as to the order in which the {@link Delete}s are executed). * @throws IOException if a remote or network exception occurs. In that case * the {@code deletes} argument will contain the {@link Delete} instances * that have not be successfully applied. * @since 0.20.1 + * @apiNote In 3.0.0 version, input list {@code deletes} will no longer be modified */ void delete(List deletes) throws IOException; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index a898abbf242719afba180ad3e26daa9478c506c4..4e2b65ad0cfcaba0b5b905f080d3acadbd025185 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -76,8 +76,6 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionObserver; import org.apache.hadoop.hbase.exceptions.ScannerResetException; import org.apache.hadoop.hbase.filter.BinaryComparator; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; @@ -116,9 +114,7 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; -import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Rule; @@ -139,6 +135,7 @@ public class TestFromClientSide { protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static byte [] ROW = Bytes.toBytes("testRow"); private static byte [] FAMILY = Bytes.toBytes("testFamily"); + private static final byte[] INVALID_FAMILY = Bytes.toBytes("invalidTestFamily"); private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); private static byte [] VALUE = Bytes.toBytes("testValue"); protected static int SLAVES = 3; @@ -174,22 +171,6 @@ public class TestFromClientSide { } /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - // Nothing to do. - } - - /** - * @throws java.lang.Exception - */ - @After - public void tearDown() throws Exception { - // Nothing to do. - } - - /** * Test append result when there are duplicate rpc request. */ @Test @@ -2377,6 +2358,115 @@ public class TestFromClientSide { } } + /** + * Test batch operations with combination of valid and invalid args + */ + @Test + public void testBatchOperationsWithErrors() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10); + + int NUM_OPS = 100; + int FAILED_OPS = 50; + + RetriesExhaustedWithDetailsException expectedException = null; + IllegalArgumentException iae = null; + + // 1.1 Put with no column families (local validation, runtime exception) + List puts = new ArrayList(NUM_OPS); + for (int i = 0; i != NUM_OPS; i++) { + Put put = new Put(Bytes.toBytes(i)); + puts.add(put); + } + + try { + foo.put(puts); + } catch (IllegalArgumentException e) { + iae = e; + } + assertNotNull(iae); + assertEquals(NUM_OPS, puts.size()); + + // 1.2 Put with invalid column family + iae = null; + puts.clear(); + for (int i = 0; i != NUM_OPS; i++) { + Put put = new Put(Bytes.toBytes(i)); + put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i)); + puts.add(put); + } + + try { + foo.put(puts); + } catch (RetriesExhaustedWithDetailsException e) { + expectedException = e; + } + assertNotNull(expectedException); + assertEquals(FAILED_OPS, expectedException.exceptions.size()); + assertTrue(expectedException.actions.contains(puts.get(1))); + + // 2.1 Get non-existent rows + List gets = new ArrayList<>(NUM_OPS); + for (int i = 0; i < NUM_OPS; i++) { + Get get = new Get(Bytes.toBytes(i)); + // get.addColumn(FAMILY, FAMILY); + gets.add(get); + } + Result[] getsResult = foo.get(gets); + + assertNotNull(getsResult); + assertEquals(NUM_OPS, getsResult.length); + assertNull(getsResult[1].getRow()); + + // 2.2 Get with invalid column family + gets.clear(); + getsResult = null; + expectedException = null; + for (int i = 0; i < NUM_OPS; i++) { + Get get = new Get(Bytes.toBytes(i)); + get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY); + gets.add(get); + } + try { + getsResult = foo.get(gets); + } catch (RetriesExhaustedWithDetailsException e) { + expectedException = e; + } + assertNull(getsResult); + assertNotNull(expectedException); + assertEquals(FAILED_OPS, expectedException.exceptions.size()); + assertTrue(expectedException.actions.contains(gets.get(1))); + + // 3.1 Delete with invalid column family + expectedException = null; + List deletes = new ArrayList<>(NUM_OPS); + for (int i = 0; i < NUM_OPS; i++) { + Delete delete = new Delete(Bytes.toBytes(i)); + delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY); + deletes.add(delete); + } + try { + foo.delete(deletes); + } catch (RetriesExhaustedWithDetailsException e) { + expectedException = e; + } + assertEquals((NUM_OPS - FAILED_OPS), deletes.size()); + assertNotNull(expectedException); + assertEquals(FAILED_OPS, expectedException.exceptions.size()); + assertTrue(expectedException.actions.contains(deletes.get(1))); + + + // 3.2 Delete non-existent rows + deletes.clear(); + for (int i = 0; i < NUM_OPS; i++) { + Delete delete = new Delete(Bytes.toBytes(i)); + deletes.add(delete); + } + foo.delete(deletes); + + assertTrue(deletes.isEmpty()); + } + /* * Baseline "scalability" test. * @@ -5421,7 +5511,7 @@ public class TestFromClientSide { List regionsInRange = new ArrayList<>(); byte[] currentKey = startKey; final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW); - try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName);) { + try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) { do { HRegionLocation regionLocation = r.getRegionLocation(currentKey); regionsInRange.add(regionLocation); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 9cbc1973de313bbd38febf1fe45d5329755f26fa..51424370838bac83d9e485a6143b2a94e96678c1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -22,17 +22,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.master.RegionState.State; -- 2.10.1 (Apple Git-78)