.../java/org/apache/hadoop/hbase/client/Admin.java | 108 ++++++++++++++++++--- .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 91 +++++++++++------ .../hadoop/hbase/client/TestSnapshotFromAdmin.java | 24 ++--- .../hbase/tmpl/master/MasterStatusTmpl.jamon | 4 +- .../hadoop/hbase/snapshot/CreateSnapshot.java | 4 +- .../apache/hadoop/hbase/snapshot/SnapshotInfo.java | 50 +++++++--- .../resources/hbase-webapps/master/snapshot.jsp | 4 +- .../main/resources/hbase-webapps/master/table.jsp | 2 +- .../hadoop/hbase/client/TestFromClientSide.java | 3 +- .../hbase/client/TestSnapshotFromClient.java | 41 ++++---- .../hbase/master/TestDistributedLogSplitting.java | 5 +- .../hadoop/hbase/master/TestWarmupRegion.java | 4 +- .../procedure/TestCloneSnapshotProcedure.java | 5 +- .../procedure/TestRestoreSnapshotProcedure.java | 4 +- .../hbase/mob/compactions/TestMobCompactor.java | 8 +- .../hbase/regionserver/TestCompactionState.java | 26 ++--- .../apache/hadoop/hbase/regionserver/TestTags.java | 7 +- .../hbase/snapshot/SnapshotTestingUtils.java | 39 +++++--- .../snapshot/TestFlushSnapshotFromClient.java | 66 ++++++++----- .../TestRestoreFlushSnapshotFromClient.java | 8 +- 20 files changed, 331 insertions(+), 172 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 54d2cb9..21a7f33 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -41,9 +41,6 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.security.SecurityCapability; import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.quotas.QuotaFilter; import org.apache.hadoop.hbase.quotas.QuotaRetriever; import org.apache.hadoop.hbase.quotas.QuotaSettings; @@ -1153,7 +1150,7 @@ public interface Admin extends Abortable, Closeable { * @return the current compaction state * @throws IOException if a remote or network exception occurs */ - AdminProtos.GetRegionInfoResponse.CompactionState getCompactionState(final TableName tableName) + Admin.CompactionState getCompactionState(final TableName tableName) throws IOException; /** @@ -1164,7 +1161,7 @@ public interface Admin extends Abortable, Closeable { * @return the current compaction state * @throws IOException if a remote or network exception occurs */ - AdminProtos.GetRegionInfoResponse.CompactionState getCompactionStateForRegion( + Admin.CompactionState getCompactionStateForRegion( final byte[] regionName) throws IOException; /** @@ -1244,7 +1241,7 @@ public interface Admin extends Abortable, Closeable { */ void snapshot(final String snapshotName, final TableName tableName, - HBaseProtos.SnapshotDescription.Type type) throws IOException, SnapshotCreationException, + Admin.SnapshotType type) throws IOException, SnapshotCreationException, IllegalArgumentException; /** @@ -1265,7 +1262,7 @@ public interface Admin extends Abortable, Closeable { * @throws SnapshotCreationException if snapshot failed to be taken * @throws IllegalArgumentException if the snapshot request is formatted incorrectly */ - void snapshot(HBaseProtos.SnapshotDescription snapshot) + void snapshot(Admin.SnapshotDescription snapshot) throws IOException, SnapshotCreationException, IllegalArgumentException; /** @@ -1278,7 +1275,7 @@ public interface Admin extends Abortable, Closeable { * @throws SnapshotCreationException if snapshot creation failed * @throws IllegalArgumentException if the snapshot request is formatted incorrectly */ - MasterProtos.SnapshotResponse takeSnapshotAsync(HBaseProtos.SnapshotDescription snapshot) + void takeSnapshotAsync(SnapshotDescription snapshot) throws IOException, SnapshotCreationException; /** @@ -1297,7 +1294,7 @@ public interface Admin extends Abortable, Closeable { * @throws org.apache.hadoop.hbase.snapshot.UnknownSnapshotException if the requested snapshot is * unknown */ - boolean isSnapshotFinished(final HBaseProtos.SnapshotDescription snapshot) + boolean isSnapshotFinished(final Admin.SnapshotDescription snapshot) throws IOException, HBaseSnapshotException, UnknownSnapshotException; /** @@ -1470,7 +1467,7 @@ public interface Admin extends Abortable, Closeable { * @return a list of snapshot descriptors for completed snapshots * @throws IOException if a network error occurs */ - List listSnapshots() throws IOException; + List listSnapshots() throws IOException; /** * List all the completed snapshots matching the given regular expression. @@ -1479,7 +1476,7 @@ public interface Admin extends Abortable, Closeable { * @return - returns a List of SnapshotDescription * @throws IOException if a remote or network exception occurs */ - List listSnapshots(String regex) throws IOException; + List listSnapshots(String regex) throws IOException; /** * List all the completed snapshots matching the given pattern. @@ -1488,7 +1485,7 @@ public interface Admin extends Abortable, Closeable { * @return - returns a List of SnapshotDescription * @throws IOException if a remote or network exception occurs */ - List listSnapshots(Pattern pattern) throws IOException; + List listSnapshots(Pattern pattern) throws IOException; /** * List all the completed snapshots matching the given table name regular expression and snapshot @@ -1498,7 +1495,7 @@ public interface Admin extends Abortable, Closeable { * @return - returns a List of completed SnapshotDescription * @throws IOException if a remote or network exception occurs */ - List listTableSnapshots(String tableNameRegex, + List listTableSnapshots(String tableNameRegex, String snapshotNameRegex) throws IOException; /** @@ -1509,7 +1506,7 @@ public interface Admin extends Abortable, Closeable { * @return - returns a List of completed SnapshotDescription * @throws IOException if a remote or network exception occurs */ - List listTableSnapshots(Pattern tableNamePattern, + List listTableSnapshots(Pattern tableNamePattern, Pattern snapshotNamePattern) throws IOException; /** @@ -1701,7 +1698,7 @@ public interface Admin extends Abortable, Closeable { * @return the current compaction state * @throws IOException if a remote or network exception occurs */ - AdminProtos.GetRegionInfoResponse.CompactionState getCompactionState(final TableName tableName, + Admin.CompactionState getCompactionState(final TableName tableName, CompactType compactType) throws IOException; /** @@ -1751,4 +1748,85 @@ public interface Admin extends Abortable, Closeable { MERGE } + @InterfaceAudience.Public + @InterfaceStability.Evolving + /** + * The POJO equivalent of HBaseProtos.SnapshotDescription + */ + public class SnapshotDescription { + private String name; + private String table; + private SnapshotType snapShotType = SnapshotType.DISABLED; + private String owner; + private long creationTime = -1l; + private int version = -1; + + public SnapshotDescription(String name) { + this(name, null); + } + + public SnapshotDescription(String name, String table) { + this(name, table, SnapshotType.DISABLED, null); + } + + public SnapshotDescription(String name, String table, SnapshotType type) { + this(name, table, type, null); + } + + public SnapshotDescription(String name, String table, SnapshotType type, String owner) { + this(name, table, type, owner, -1, -1); + } + + public SnapshotDescription(String name, String table, SnapshotType type, String owner, + long creationTime, int version) { + this.name = name; + this.table = table; + this.snapShotType = type; + this.owner = owner; + this.creationTime = creationTime; + this.version = version; + } + + public String getName() { + return this.name; + } + + public String getTable() { + return this.table; + } + + public SnapshotType getType() { + return this.snapShotType; + } + + public String getOwner() { + return this.owner; + } + + public long getCreationTime() { + return this.creationTime; + } + + public int getVersion() { + return this.version; + } + } + + @InterfaceAudience.Public + @InterfaceStability.Evolving + /** + * POJO representing the snapshot type + */ + public enum SnapshotType { + DISABLED, FLUSH, SKIPFLUSH; + } + + @InterfaceAudience.Public + @InterfaceStability.Evolving + /** + * POJO representing the compaction state + */ + public enum CompactionState { + NONE, MINOR, MAJOR, MAJOR_AND_MINOR; + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 9541967..5f86264 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -73,6 +73,7 @@ import org.apache.hadoop.hbase.ipc.RegionServerCoprocessorRpcChannel; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse; @@ -80,7 +81,6 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionReque import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest; @@ -89,7 +89,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AbortProcedureRequest; @@ -2262,7 +2261,11 @@ public class HBaseAdmin implements Admin { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); // TODO: this does not do retries, it should. Set priority and timeout in controller GetRegionInfoResponse response = admin.getRegionInfo(controller, request); - return response.getCompactionState(); + if (response.getCompactionState() != null) { + return Admin.CompactionState.valueOf(response.getCompactionState().toString()); + } else { + return null; + } } catch (ServiceException se) { throw ProtobufUtil.getRemoteException(se); } @@ -2272,31 +2275,28 @@ public class HBaseAdmin implements Admin { public void snapshot(final String snapshotName, final TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { - snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH); + snapshot(snapshotName, tableName, SnapshotType.FLUSH); } @Override public void snapshot(final byte[] snapshotName, final TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { - snapshot(Bytes.toString(snapshotName), tableName, SnapshotDescription.Type.FLUSH); + snapshot(Bytes.toString(snapshotName), tableName, Admin.SnapshotType.FLUSH); } @Override public void snapshot(final String snapshotName, final TableName tableName, - SnapshotDescription.Type type) + Admin.SnapshotType type) throws IOException, SnapshotCreationException, IllegalArgumentException { - SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); - builder.setTable(tableName.getNameAsString()); - builder.setName(snapshotName); - builder.setType(type); - snapshot(builder.build()); + snapshot(new Admin.SnapshotDescription(snapshotName, tableName.getNameAsString(), type)); } @Override - public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException, + public void snapshot(SnapshotDescription snapshotDesc) throws IOException, SnapshotCreationException, IllegalArgumentException { // actually take the snapshot - SnapshotResponse response = takeSnapshotAsync(snapshot); + HBaseProtos.SnapshotDescription snapshot = createHBaseProtosSnapshotDesc(snapshotDesc); + SnapshotResponse response = asyncSnapshot(snapshot); final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot) .build(); IsSnapshotDoneResponse done = null; @@ -2336,8 +2336,25 @@ public class HBaseAdmin implements Admin { } @Override - public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException, + public void takeSnapshotAsync(SnapshotDescription snapshotDesc) throws IOException, SnapshotCreationException { + HBaseProtos.SnapshotDescription snapshot = createHBaseProtosSnapshotDesc(snapshotDesc); + asyncSnapshot(snapshot); + } + + private HBaseProtos.SnapshotDescription + createHBaseProtosSnapshotDesc(SnapshotDescription snapshotDesc) { + HBaseProtos.SnapshotDescription.Builder builder = HBaseProtos.SnapshotDescription.newBuilder(); + builder.setTable(snapshotDesc.getTable()); + builder.setName(snapshotDesc.getName()); + builder.setType( + HBaseProtos.SnapshotDescription.Type.valueOf(snapshotDesc.getType().name())); + HBaseProtos.SnapshotDescription snapshot = builder.build(); + return snapshot; + } + + private SnapshotResponse asyncSnapshot(HBaseProtos.SnapshotDescription snapshot) + throws IOException { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot) .build(); @@ -2353,9 +2370,9 @@ public class HBaseAdmin implements Admin { } @Override - public boolean isSnapshotFinished(final SnapshotDescription snapshot) + public boolean isSnapshotFinished(final SnapshotDescription snapshotDesc) throws IOException, HBaseSnapshotException, UnknownSnapshotException { - + final HBaseProtos.SnapshotDescription snapshot = createHBaseProtosSnapshotDesc(snapshotDesc); return executeCallable(new MasterCallable(getConnection()) { @Override public IsSnapshotDoneResponse call(int callTimeout) throws ServiceException { @@ -2640,7 +2657,7 @@ public class HBaseAdmin implements Admin { private Future internalRestoreSnapshotAsync( final String snapshotName, final TableName tableName) throws IOException, RestoreSnapshotException { - final SnapshotDescription snapshot = SnapshotDescription.newBuilder() + final HBaseProtos.SnapshotDescription snapshot = HBaseProtos.SnapshotDescription.newBuilder() .setName(snapshotName).setTable(tableName.getNameAsString()).build(); // actually restore the snapshot @@ -2668,7 +2685,7 @@ public class HBaseAdmin implements Admin { private static class RestoreSnapshotFuture extends TableFuture { public RestoreSnapshotFuture( final HBaseAdmin admin, - final SnapshotDescription snapshot, + final HBaseProtos.SnapshotDescription snapshot, final TableName tableName, final RestoreSnapshotResponse response) { super(admin, tableName, @@ -2699,8 +2716,16 @@ public class HBaseAdmin implements Admin { public List call(int callTimeout) throws ServiceException { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setCallTimeout(callTimeout); - return master.getCompletedSnapshots(controller, - GetCompletedSnapshotsRequest.newBuilder().build()).getSnapshotsList(); + List snapshotsList = master + .getCompletedSnapshots(controller, GetCompletedSnapshotsRequest.newBuilder().build()) + .getSnapshotsList(); + List result = new ArrayList(snapshotsList.size()); + for (HBaseProtos.SnapshotDescription snapshot : snapshotsList) { + result.add(new SnapshotDescription(snapshot.getName(), snapshot.getTable(), + SnapshotType.valueOf(snapshot.getType().toString()), snapshot.getOwner(), + snapshot.getCreationTime(), snapshot.getVersion())); + } + return result; } }); } @@ -2762,7 +2787,9 @@ public class HBaseAdmin implements Admin { controller.setCallTimeout(callTimeout); master.deleteSnapshot(controller, DeleteSnapshotRequest.newBuilder(). - setSnapshot(SnapshotDescription.newBuilder().setName(snapshotName).build()).build() + setSnapshot( + HBaseProtos.SnapshotDescription.newBuilder().setName(snapshotName).build()) + .build() ); return null; } @@ -2795,7 +2822,7 @@ public class HBaseAdmin implements Admin { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setCallTimeout(callTimeout); this.master.deleteSnapshot(controller, DeleteSnapshotRequest.newBuilder() - .setSnapshot(snapshot).build()); + .setSnapshot(createHBaseProtosSnapshotDesc(snapshot)).build()); return null; } }); @@ -2996,9 +3023,10 @@ public class HBaseAdmin implements Admin { * {@inheritDoc} */ @Override - public CompactionState getCompactionState(TableName tableName, + public Admin.CompactionState getCompactionState(TableName tableName, CompactType compactType) throws IOException { - CompactionState state = CompactionState.NONE; + AdminProtos.GetRegionInfoResponse.CompactionState state = + AdminProtos.GetRegionInfoResponse.CompactionState.NONE; checkTableExists(tableName); PayloadCarryingRpcController controller = rpcControllerFactory.newController(); switch (compactType) { @@ -3040,16 +3068,16 @@ public class HBaseAdmin implements Admin { case MAJOR_AND_MINOR: return CompactionState.MAJOR_AND_MINOR; case MAJOR: - if (state == CompactionState.MINOR) { + if (state == AdminProtos.GetRegionInfoResponse.CompactionState.MINOR) { return CompactionState.MAJOR_AND_MINOR; } - state = CompactionState.MAJOR; + state = AdminProtos.GetRegionInfoResponse.CompactionState.MAJOR; break; case MINOR: - if (state == CompactionState.MAJOR) { - return CompactionState.MAJOR_AND_MINOR; + if (state == AdminProtos.GetRegionInfoResponse.CompactionState.MAJOR) { + return Admin.CompactionState.MAJOR_AND_MINOR; } - state = CompactionState.MINOR; + state = AdminProtos.GetRegionInfoResponse.CompactionState.MINOR; break; case NONE: default: // nothing, continue @@ -3080,7 +3108,10 @@ public class HBaseAdmin implements Admin { } break; } - return state; + if(state != null) { + return Admin.CompactionState.valueOf(state.toString()); + } + return null; } /** diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java index 4d55c33..53383f6 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java @@ -141,18 +141,17 @@ public class TestSnapshotFromAdmin { Mockito.when(mockConnection.getRpcRetryingCallerFactory()).thenReturn(callerFactory); Mockito.when(mockConnection.getRpcControllerFactory()).thenReturn(controllerFactory); Admin admin = new HBaseAdmin(mockConnection); - SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); // check that invalid snapshot names fail - failSnapshotStart(admin, builder.setName(HConstants.SNAPSHOT_DIR_NAME).build()); - failSnapshotStart(admin, builder.setName("-snapshot").build()); - failSnapshotStart(admin, builder.setName("snapshot fails").build()); - failSnapshotStart(admin, builder.setName("snap$hot").build()); - failSnapshotStart(admin, builder.setName("snap:hot").build()); + failSnapshotStart(admin, new Admin.SnapshotDescription(HConstants.SNAPSHOT_DIR_NAME)); + failSnapshotStart(admin, new Admin.SnapshotDescription("-snapshot")); + failSnapshotStart(admin, new Admin.SnapshotDescription("snapshot fails")); + failSnapshotStart(admin, new Admin.SnapshotDescription("snap$hot")); + failSnapshotStart(admin, new Admin.SnapshotDescription("snap:hot")); // check the table name also get verified - failSnapshotStart(admin, builder.setName("snapshot").setTable(".table").build()); - failSnapshotStart(admin, builder.setName("snapshot").setTable("-table").build()); - failSnapshotStart(admin, builder.setName("snapshot").setTable("table fails").build()); - failSnapshotStart(admin, builder.setName("snapshot").setTable("tab%le").build()); + failSnapshotStart(admin, new Admin.SnapshotDescription("snapshot", ".table")); + failSnapshotStart(admin, new Admin.SnapshotDescription("snapshot", "-table")); + failSnapshotStart(admin, new Admin.SnapshotDescription("snapshot", "table fails")); + failSnapshotStart(admin, new Admin.SnapshotDescription("snapshot", "tab%le")); // mock the master connection MasterKeepAliveConnection master = Mockito.mock(MasterKeepAliveConnection.class); @@ -167,10 +166,11 @@ public class TestSnapshotFromAdmin { Mockito.any(IsSnapshotDoneRequest.class))).thenReturn(doneResponse); // make sure that we can use valid names - admin.snapshot(builder.setName("snapshot").setTable("table").build()); + admin.snapshot(new Admin.SnapshotDescription("snapshot", "table")); } - private void failSnapshotStart(Admin admin, SnapshotDescription snapshot) throws IOException { + private void failSnapshotStart(Admin admin, Admin.SnapshotDescription snapshot) + throws IOException { try { admin.snapshot(snapshot); fail("Snapshot should not have succeed with name:" + snapshot.getName()); diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index 951a95e..ffdda14 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -454,7 +454,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); <%def userSnapshots> <%java> - List snapshots = null; + List snapshots = null; try (Admin admin = master.getConnection().getAdmin()) { snapshots = master.isInitialized() ? admin.listSnapshots() : null; } @@ -466,7 +466,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); Table Creation Time - <%for SnapshotDescription snapshotDesc : snapshots%> + <%for Admin.SnapshotDescription snapshotDesc : snapshots%> <%java> TableName snapshotTable = TableName.valueOf(snapshotDesc.getTable()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CreateSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CreateSnapshot.java index 06b6017..b64fa85 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CreateSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CreateSnapshot.java @@ -68,8 +68,8 @@ public class CreateSnapshot extends AbstractHBaseTool { if (snapshotType != null) { type = HBaseProtos.SnapshotDescription.Type.valueOf(snapshotName.toUpperCase()); } - - admin.snapshot(snapshotName, TableName.valueOf(tableName), type); + admin.snapshot(new Admin.SnapshotDescription(snapshotName, tableName, + Admin.SnapshotType.valueOf(type.toString()))); } catch (Exception e) { return -1; } finally { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java index 0a359f8..0885f41 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; @@ -47,6 +48,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.WALLink; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.util.FSUtils; @@ -131,17 +133,20 @@ public final class SnapshotInfo extends Configured implements Tool { private final Configuration conf; private final FileSystem fs; - SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot) + SnapshotStats(final Configuration conf, final FileSystem fs, + final Admin.SnapshotDescription snapshot) { - this.snapshot = snapshot; + this.snapshot = createHBaseProtosSnapshotDesc(snapshot); this.snapshotTable = TableName.valueOf(snapshot.getTable()); this.conf = conf; this.fs = fs; } /** @return the snapshot descriptor */ - public SnapshotDescription getSnapshotDescription() { - return this.snapshot; + public Admin.SnapshotDescription getSnapshotDescription() { + return new Admin.SnapshotDescription(this.snapshot.getName(), this.snapshot.getTable(), + Admin.SnapshotType.valueOf(this.snapshot.getType().toString()), this.snapshot.getOwner(), + this.snapshot.getCreationTime(), this.snapshot.getVersion()); } /** @return true if the snapshot is corrupted */ @@ -327,7 +332,7 @@ public final class SnapshotInfo extends Configured implements Tool { if (listSnapshots) { SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME"); - for (SnapshotDescription desc: getSnapshotList(conf)) { + for (Admin.SnapshotDescription desc: getSnapshotList(conf)) { System.out.printf("%-20s | %20s | %s%n", desc.getName(), df.format(new Date(desc.getCreationTime())), @@ -415,7 +420,10 @@ public final class SnapshotInfo extends Configured implements Tool { // Collect information about hfiles and logs in the snapshot final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); final String table = snapshotDesc.getTable(); - final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc); + Admin.SnapshotDescription desc = new Admin.SnapshotDescription(snapshotDesc.getName(), + snapshotDesc.getTable(), Admin.SnapshotType.valueOf(snapshotDesc.getType().toString()), + snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion()); + final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc); SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest, new SnapshotReferenceUtil.SnapshotVisitor() { @Override @@ -491,11 +499,12 @@ public final class SnapshotInfo extends Configured implements Tool { * @return the snapshot stats */ public static SnapshotStats getSnapshotStats(final Configuration conf, - final SnapshotDescription snapshot) throws IOException { + final Admin.SnapshotDescription snapshot) throws IOException { + SnapshotDescription snapshotDesc = createHBaseProtosSnapshotDesc(snapshot); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); - Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); - SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir); + SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot); SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, new SnapshotReferenceUtil.SnapshotVisitor() { @@ -515,17 +524,21 @@ public final class SnapshotInfo extends Configured implements Tool { * @param conf the {@link Configuration} to use * @return the list of snapshots */ - public static List getSnapshotList(final Configuration conf) + public static List getSnapshotList(final Configuration conf) throws IOException { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); FileStatus[] snapshots = fs.listStatus(snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); - List snapshotLists = - new ArrayList(snapshots.length); + List snapshotLists = + new ArrayList(snapshots.length); for (FileStatus snapshotDirStat: snapshots) { - snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath())); + SnapshotDescription snapshotDesc = + SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()); + snapshotLists.add(new Admin.SnapshotDescription(snapshotDesc.getName(), + snapshotDesc.getTable(), Admin.SnapshotType.valueOf(snapshotDesc.getType().toString()), + snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion())); } return snapshotLists; } @@ -544,4 +557,15 @@ public final class SnapshotInfo extends Configured implements Tool { public static void main(String[] args) throws Exception { System.exit(innerMain(args)); } + + private static HBaseProtos.SnapshotDescription + createHBaseProtosSnapshotDesc(Admin.SnapshotDescription snapshotDesc) { + HBaseProtos.SnapshotDescription.Builder builder = HBaseProtos.SnapshotDescription.newBuilder(); + builder.setTable(snapshotDesc.getTable()); + builder.setName(snapshotDesc.getName()); + builder.setType( + HBaseProtos.SnapshotDescription.Type.valueOf(snapshotDesc.getType().name())); + HBaseProtos.SnapshotDescription snapshot = builder.build(); + return snapshot; + } } diff --git a/hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp b/hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp index 5900383..9f5ef2d 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp @@ -32,12 +32,12 @@ Configuration conf = master.getConfiguration(); boolean readOnly = conf.getBoolean("hbase.master.ui.readonly", false); String snapshotName = request.getParameter("name"); - SnapshotDescription snapshot = null; + Admin.SnapshotDescription snapshot = null; SnapshotInfo.SnapshotStats stats = null; TableName snapshotTable = null; boolean tableExists = false; try (Admin admin = master.getConnection().getAdmin()) { - for (SnapshotDescription snapshotDesc: admin.listSnapshots()) { + for (Admin.SnapshotDescription snapshotDesc: admin.listSnapshots()) { if (snapshotName.equals(snapshotDesc.getName())) { snapshot = snapshotDesc; stats = SnapshotInfo.getSnapshotStats(conf, snapshot); diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 4a151e7..09a5185 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -222,7 +222,7 @@ if ( fqtn != null ) { <% try { - CompactionState compactionState = admin.getCompactionState(table.getName()); + Admin.CompactionState compactionState = admin.getCompactionState(table.getName()); %> <%= compactionState %> <% diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index 520f210..38c568a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -85,7 +85,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType; import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService; @@ -278,7 +277,7 @@ public class TestFromClientSide { @Override public boolean evaluate() throws IOException { return TEST_UTIL.getHBaseAdmin().getCompactionState(TABLENAME) == - AdminProtos.GetRegionInfoResponse.CompactionState.NONE; + Admin.CompactionState.NONE; } }); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java index 1b61147..2e95267 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java @@ -35,8 +35,8 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.Admin.SnapshotType; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; @@ -176,7 +176,7 @@ public class TestSnapshotFromClient { // delete the first two snapshots admin.deleteSnapshots("TableSnapshot.*"); - List snapshots = admin.listSnapshots(); + List snapshots = admin.listSnapshots(); assertEquals(1, snapshots.size()); assertEquals(snapshots.get(0).getName(), snapshot3); @@ -212,18 +212,13 @@ public class TestSnapshotFromClient { final String SNAPSHOT_NAME = "offlineTableSnapshot"; byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME); - SnapshotDescription desc = SnapshotDescription.newBuilder() - .setType(SnapshotDescription.Type.DISABLED) - .setTable(STRING_TABLE_NAME) - .setName(SNAPSHOT_NAME) - .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION) - .build(); - admin.snapshot(desc); + admin.snapshot(new Admin.SnapshotDescription(SNAPSHOT_NAME, STRING_TABLE_NAME, + SnapshotType.DISABLED, null, -1, SnapshotManifestV1.DESCRIPTOR_VERSION)); LOG.debug("Snapshot completed."); // make sure we have the snapshot - List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, - snapshot, TABLE_NAME); + List snapshots = + SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME); // make sure its a valid snapshot FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); @@ -231,9 +226,9 @@ public class TestSnapshotFromClient { LOG.debug("FS state after snapshot:"); FSUtils.logFileSystemState(UTIL.getTestFileSystem(), FSUtils.getRootDir(UTIL.getConfiguration()), LOG); - - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs); + SnapshotTestingUtils.confirmSnapshotValid( + SnapshotTestingUtils.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM, + rootDir, admin, fs); admin.deleteSnapshot(snapshot); snapshots = admin.listSnapshots(); @@ -292,8 +287,8 @@ public class TestSnapshotFromClient { LOG.debug("Snapshot completed."); // make sure we have the snapshot - List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, - snapshot, TABLE_NAME); + List snapshots = + SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME); // make sure its a valid snapshot FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); @@ -304,8 +299,9 @@ public class TestSnapshotFromClient { List emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region List nonEmptyCfs = Lists.newArrayList(); - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs, - rootDir, admin, fs); + SnapshotTestingUtils.confirmSnapshotValid( + SnapshotTestingUtils.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs, + emptyCfs, rootDir, admin, fs); admin.deleteSnapshot(snapshot); snapshots = admin.listSnapshots(); @@ -334,10 +330,10 @@ public class TestSnapshotFromClient { admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName2); LOG.debug(table2Snapshot1 + " completed."); - List listTableSnapshots = admin.listTableSnapshots("test.*", ".*"); + List listTableSnapshots = admin.listTableSnapshots("test.*", ".*"); List listTableSnapshotNames = new ArrayList(); assertEquals(3, listTableSnapshots.size()); - for (SnapshotDescription s : listTableSnapshots) { + for (Admin.SnapshotDescription s : listTableSnapshots) { listTableSnapshotNames.add(s.getName()); } assertTrue(listTableSnapshotNames.contains(table1Snapshot1)); @@ -375,10 +371,11 @@ public class TestSnapshotFromClient { admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME); LOG.debug(table2Snapshot1 + " completed."); - List listTableSnapshots = admin.listTableSnapshots("test.*", "Table1.*"); + List listTableSnapshots = + admin.listTableSnapshots("test.*", "Table1.*"); List listTableSnapshotNames = new ArrayList(); assertEquals(2, listTableSnapshots.size()); - for (SnapshotDescription s : listTableSnapshots) { + for (Admin.SnapshotDescription s : listTableSnapshots) { listTableSnapshotNames.add(s.getName()); } assertTrue(listTableSnapshotNames.contains(table1Snapshot1)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index cff8db0..c70a079 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -80,6 +80,7 @@ import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination; import org.apache.hadoop.hbase.exceptions.OperationConflictException; @@ -87,7 +88,6 @@ import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; @@ -1421,7 +1421,8 @@ public class TestDistributedLogSplitting { TEST_UTIL.waitFor(30000, 200, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { - return (TEST_UTIL.getHBaseAdmin().getCompactionState(tableName) == CompactionState.NONE); + return (TEST_UTIL.getHBaseAdmin() + .getCompactionState(tableName) == Admin.CompactionState.NONE); } }); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java index 53ee92b..a11ee0d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -106,7 +106,7 @@ public class TestWarmupRegion { @Override public boolean evaluate() throws IOException { return TEST_UTIL.getHBaseAdmin().getCompactionState(TABLENAME) == - AdminProtos.GetRegionInfoResponse.CompactionState.NONE; + Admin.CompactionState.NONE; } }); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java index aeafbf8..1e0b394 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CloneSnapshotState; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; @@ -115,8 +116,8 @@ public class TestCloneSnapshotProcedure { admin.snapshot(snapshotName, snapshotTableName); admin.enableTable(snapshotTableName); - List snapshotList = admin.listSnapshots(); - snapshot = snapshotList.get(0); + List snapshotList = admin.listSnapshots(); + snapshot = SnapshotTestingUtils.createHBaseProtosSnapshotDesc(snapshotList.get(0)); } return snapshot; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java index 44d6988..6f25147 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java @@ -140,8 +140,8 @@ public class TestRestoreSnapshotProcedure { // take a snapshot admin.snapshot(snapshotName, snapshotTableName); - List snapshotList = admin.listSnapshots(); - snapshot = snapshotList.get(0); + List snapshotList = admin.listSnapshots(); + snapshot = SnapshotTestingUtils.createHBaseProtosSnapshotDesc(snapshotList.get(0)); // modify the table HColumnDescriptor columnFamilyDescriptor3 = new HColumnDescriptor(CF3); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index 9922aff..0053516 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -461,9 +461,9 @@ public class TestMobCompactor { private void waitUntilCompactionFinished(TableName tableName) throws IOException, InterruptedException { long finished = EnvironmentEdgeManager.currentTime() + 60000; - CompactionState state = admin.getCompactionState(tableName); + Admin.CompactionState state = admin.getCompactionState(tableName); while (EnvironmentEdgeManager.currentTime() < finished) { - if (state == CompactionState.NONE) { + if (state == Admin.CompactionState.NONE) { break; } state = admin.getCompactionState(tableName); @@ -475,9 +475,9 @@ public class TestMobCompactor { private void waitUntilMobCompactionFinished(TableName tableName) throws IOException, InterruptedException { long finished = EnvironmentEdgeManager.currentTime() + 60000; - CompactionState state = admin.getCompactionState(tableName, Admin.CompactType.MOB); + Admin.CompactionState state = admin.getCompactionState(tableName, Admin.CompactType.MOB); while (EnvironmentEdgeManager.currentTime() < finished) { - if (state == CompactionState.NONE) { + if (state == Admin.CompactionState.NONE) { break; } state = admin.getCompactionState(tableName, Admin.CompactType.MOB); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java index 4715d53..c368e44 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java @@ -61,22 +61,22 @@ public class TestCompactionState { @Test(timeout=600000) public void testMajorCompaction() throws IOException, InterruptedException { - compaction("testMajorCompaction", 8, CompactionState.MAJOR, false); + compaction("testMajorCompaction", 8, Admin.CompactionState.MAJOR, false); } @Test(timeout=600000) public void testMinorCompaction() throws IOException, InterruptedException { - compaction("testMinorCompaction", 15, CompactionState.MINOR, false); + compaction("testMinorCompaction", 15, Admin.CompactionState.MINOR, false); } @Test(timeout=600000) public void testMajorCompactionOnFamily() throws IOException, InterruptedException { - compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true); + compaction("testMajorCompactionOnFamily", 8, Admin.CompactionState.MAJOR, true); } @Test(timeout=600000) public void testMinorCompactionOnFamily() throws IOException, InterruptedException { - compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true); + compaction("testMinorCompactionOnFamily", 15, Admin.CompactionState.MINOR, true); } @Test @@ -121,7 +121,7 @@ public class TestCompactionState { * @throws InterruptedException */ private void compaction(final String tableName, final int flushes, - final CompactionState expectedState, boolean singleFamily) + final Admin.CompactionState expectedState, boolean singleFamily) throws IOException, InterruptedException { // Create a table with regions TableName table = TableName.valueOf(tableName); @@ -138,7 +138,7 @@ public class TestCompactionState { int countBeforeSingleFamily = countStoreFilesInFamily(regions, family); assertTrue(countBefore > 0); // there should be some data files Admin admin = TEST_UTIL.getHBaseAdmin(); - if (expectedState == CompactionState.MINOR) { + if (expectedState == Admin.CompactionState.MINOR) { if (singleFamily) { admin.compact(table, family); } else { @@ -154,8 +154,8 @@ public class TestCompactionState { long curt = System.currentTimeMillis(); long waitTime = 5000; long endt = curt + waitTime; - CompactionState state = admin.getCompactionState(table); - while (state == CompactionState.NONE && curt < endt) { + Admin.CompactionState state = admin.getCompactionState(table); + while (state == Admin.CompactionState.NONE && curt < endt) { Thread.sleep(10); state = admin.getCompactionState(table); curt = System.currentTimeMillis(); @@ -164,30 +164,30 @@ public class TestCompactionState { // otherwise, the compaction should have already been done if (expectedState != state) { for (Region region: regions) { - state = region.getCompactionState(); + state = Admin.CompactionState.valueOf(region.getCompactionState().toString()); assertEquals(CompactionState.NONE, state); } } else { // Wait until the compaction is done state = admin.getCompactionState(table); - while (state != CompactionState.NONE && curt < endt) { + while (state != Admin.CompactionState.NONE && curt < endt) { Thread.sleep(10); state = admin.getCompactionState(table); } // Now, compaction should be done. - assertEquals(CompactionState.NONE, state); + assertEquals(Admin.CompactionState.NONE, state); } int countAfter = countStoreFilesInFamilies(regions, families); int countAfterSingleFamily = countStoreFilesInFamily(regions, family); assertTrue(countAfter < countBefore); if (!singleFamily) { - if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter); + if (expectedState == Admin.CompactionState.MAJOR) assertTrue(families.length == countAfter); else assertTrue(families.length < countAfter); } else { int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily; // assert only change was to single column family assertTrue(singleFamDiff == (countBefore - countAfter)); - if (expectedState == CompactionState.MAJOR) { + if (expectedState == Admin.CompactionState.MAJOR) { assertTrue(1 == countAfterSingleFamily); } else { assertTrue(1 < countAfterSingleFamily); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java index 3a9ace2..6d69ba2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java @@ -53,7 +53,6 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; @@ -152,7 +151,7 @@ public class TestTags { result(fam, row, qual, row2, table, value, value2, row1, value1); admin.compact(tableName); - while (admin.getCompactionState(tableName) != CompactionState.NONE) { + while (admin.getCompactionState(tableName) != Admin.CompactionState.NONE) { Thread.sleep(10); } result(fam, row, qual, row2, table, value, value2, row1, value1); @@ -229,7 +228,7 @@ public class TestTags { scanner.close(); } admin.compact(tableName); - while (admin.getCompactionState(tableName) != CompactionState.NONE) { + while (admin.getCompactionState(tableName) != Admin.CompactionState.NONE) { Thread.sleep(10); } s = new Scan(row); @@ -340,7 +339,7 @@ public class TestTags { } TestCoprocessorForTags.checkTagPresence = false; } - while (admin.getCompactionState(tableName) != CompactionState.NONE) { + while (admin.getCompactionState(tableName) != Admin.CompactionState.NONE) { Thread.sleep(10); } TestCoprocessorForTags.checkTagPresence = true; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index f739233..b8f90be 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.TableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Admin.SnapshotType; import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Put; @@ -56,6 +57,7 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest; @@ -95,14 +97,14 @@ public class SnapshotTestingUtils { * Make sure that there is only one snapshot returned from the master and its * name and table match the passed in parameters. */ - public static List assertExistsMatchingSnapshot( + public static List assertExistsMatchingSnapshot( Admin admin, String snapshotName, TableName tableName) throws IOException { // list the snapshot - List snapshots = admin.listSnapshots(); + List snapshots = admin.listSnapshots(); - List returnedSnapshots = new ArrayList(); - for (SnapshotDescription sd : snapshots) { + List returnedSnapshots = new ArrayList(); + for (Admin.SnapshotDescription sd : snapshots) { if (snapshotName.equals(sd.getName()) && tableName.equals(TableName.valueOf(sd.getTable()))) { returnedSnapshots.add(sd); @@ -126,11 +128,11 @@ public class SnapshotTestingUtils { * Make sure that there is only one snapshot returned from the master and its * name and table match the passed in parameters. */ - public static List assertOneSnapshotThatMatches( + public static List assertOneSnapshotThatMatches( Admin admin, String snapshotName, TableName tableName) throws IOException { // list the snapshot - List snapshots = admin.listSnapshots(); + List snapshots = admin.listSnapshots(); assertEquals("Should only have 1 snapshot", 1, snapshots.size()); assertEquals(snapshotName, snapshots.get(0).getName()); @@ -143,7 +145,7 @@ public class SnapshotTestingUtils { * Make sure that there is only one snapshot returned from the master and its * name and table match the passed in parameters. */ - public static List assertOneSnapshotThatMatches( + public static List assertOneSnapshotThatMatches( Admin admin, byte[] snapshot, TableName tableName) throws IOException { return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), tableName); @@ -288,7 +290,8 @@ public class SnapshotTestingUtils { CorruptedSnapshotException lastEx = null; while (tries++ < numTries) { try { - admin.snapshot(snapshotName, TableName.valueOf(tableName), type); + admin.snapshot(new Admin.SnapshotDescription(snapshotName, tableName, + SnapshotType.valueOf(type.toString()))); return; } catch (CorruptedSnapshotException cse) { LOG.warn("Got CorruptedSnapshotException", cse); @@ -390,14 +393,24 @@ public class SnapshotTestingUtils { } admin.snapshot(snapshotNameString, tableName); - List snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, - snapshotNameString, tableName); + List snapshots = + SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName); if (snapshots == null || snapshots.size() != 1) { Assert.fail("Incorrect number of snapshots for table " + tableName); } - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames, - emptyFamilyNames, rootDir, admin, fs); + SnapshotTestingUtils.confirmSnapshotValid(createHBaseProtosSnapshotDesc(snapshots.get(0)), + tableName, nonEmptyFamilyNames, emptyFamilyNames, rootDir, admin, fs); + } + + public static HBaseProtos.SnapshotDescription + createHBaseProtosSnapshotDesc(Admin.SnapshotDescription snapshotDesc) { + HBaseProtos.SnapshotDescription.Builder builder = HBaseProtos.SnapshotDescription.newBuilder(); + builder.setTable(snapshotDesc.getTable()); + builder.setName(snapshotDesc.getName()); + builder.setType(HBaseProtos.SnapshotDescription.Type.valueOf(snapshotDesc.getType().name())); + HBaseProtos.SnapshotDescription snapshot = builder.build(); + return snapshot; } /** @@ -731,7 +744,7 @@ public class SnapshotTestingUtils { public static void deleteAllSnapshots(final Admin admin) throws IOException { // Delete all the snapshots - for (SnapshotDescription snapshot: admin.listSnapshots()) { + for (Admin.SnapshotDescription snapshot: admin.listSnapshots()) { admin.deleteSnapshot(snapshot.getName()); } SnapshotTestingUtils.assertNoSnapshots(admin); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index 4b988a6..8bdd343 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -147,18 +147,20 @@ public class TestFlushSnapshotFromClient { // take a snapshot of the enabled table String snapshotString = "offlineTableSnapshot"; byte[] snapshot = Bytes.toBytes(snapshotString); - admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH); + admin.snapshot(snapshotString, TABLE_NAME, + Admin.SnapshotType.valueOf(SnapshotDescription.Type.FLUSH.toString())); LOG.debug("Snapshot completed."); // make sure we have the snapshot - List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, - snapshot, TABLE_NAME); + List snapshots = + SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME); // make sure its a valid snapshot LOG.debug("FS state after snapshot:"); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); - SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); + SnapshotTestingUtils.confirmSnapshotValid(UTIL, + SnapshotTestingUtils.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM); } /** @@ -181,18 +183,20 @@ public class TestFlushSnapshotFromClient { // take a snapshot of the enabled table String snapshotString = "skipFlushTableSnapshot"; byte[] snapshot = Bytes.toBytes(snapshotString); - admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH); + admin.snapshot(snapshotString, TABLE_NAME, + Admin.SnapshotType.valueOf(SnapshotDescription.Type.SKIPFLUSH.toString())); LOG.debug("Snapshot completed."); // make sure we have the snapshot - List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, - snapshot, TABLE_NAME); + List snapshots = + SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME); // make sure its a valid snapshot LOG.debug("FS state after snapshot:"); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); - SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); + SnapshotTestingUtils.confirmSnapshotValid(UTIL, + SnapshotTestingUtils.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM); admin.deleteSnapshot(snapshot); snapshots = admin.listSnapshots(); @@ -227,14 +231,15 @@ public class TestFlushSnapshotFromClient { LOG.debug("Snapshot completed."); // make sure we have the snapshot - List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, + List snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME); // make sure its a valid snapshot LOG.debug("FS state after snapshot:"); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); - SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM); + SnapshotTestingUtils.confirmSnapshotValid(UTIL, + SnapshotTestingUtils.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM); } @Test @@ -258,7 +263,8 @@ public class TestFlushSnapshotFromClient { // snapshot the non-existant table try { - admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH); + admin.snapshot("fail", tableName, + Admin.SnapshotType.valueOf(SnapshotDescription.Type.FLUSH.toString())); fail("Snapshot succeeded even though there is not table."); } catch (SnapshotCreationException e) { LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage()); @@ -273,7 +279,9 @@ public class TestFlushSnapshotFromClient { .build(); // take the snapshot async - admin.takeSnapshotAsync(snapshot); + admin.takeSnapshotAsync( + new Admin.SnapshotDescription("asyncSnapshot", TABLE_NAME.getNameAsString(), + Admin.SnapshotType.valueOf(SnapshotDescription.Type.FLUSH.toString()))); // constantly loop, looking for the snapshot to complete HMaster master = UTIL.getMiniHBaseCluster().getMaster(); @@ -295,7 +303,8 @@ public class TestFlushSnapshotFromClient { // Take a snapshot String snapshotBeforeMergeName = "snapshotBeforeMerge"; - admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, SnapshotDescription.Type.FLUSH); + admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, + Admin.SnapshotType.valueOf(SnapshotDescription.Type.FLUSH.toString())); // Clone the table TableName cloneBeforeMergeName = TableName.valueOf("cloneBeforeMerge"); @@ -417,34 +426,39 @@ public class TestFlushSnapshotFromClient { final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum); // We'll have one of these per thread class SSRunnable implements Runnable { - SnapshotDescription ss; - SSRunnable(SnapshotDescription ss) { + Admin.SnapshotDescription ss; + SSRunnable(Admin.SnapshotDescription ss) { this.ss = ss; } @Override public void run() { try { - LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss)); + LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils + .toString(SnapshotTestingUtils.createHBaseProtosSnapshotDesc(ss))); admin.takeSnapshotAsync(ss); } catch (Exception e) { LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString( - ss) + SnapshotTestingUtils.createHBaseProtosSnapshotDesc(ss)) + ". This is ok, we expect some", e); } - LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss)); + LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils + .toString(SnapshotTestingUtils.createHBaseProtosSnapshotDesc(ss))); toBeSubmitted.countDown(); } }; // build descriptions - SnapshotDescription[] descs = new SnapshotDescription[ssNum]; + Admin.SnapshotDescription[] descs = new Admin.SnapshotDescription[ssNum]; for (int i = 0; i < ssNum; i++) { SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); - builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString()); - builder.setName("ss"+i); - builder.setType(SnapshotDescription.Type.FLUSH); - descs[i] = builder.build(); + if(i %2 ==0) { + descs[i] = new Admin.SnapshotDescription("ss" + i, TABLE_NAME.getNameAsString(), + Admin.SnapshotType.valueOf(SnapshotDescription.Type.FLUSH.toString())); + } else { + descs[i] = new Admin.SnapshotDescription("ss" + i, TABLE2_NAME.getNameAsString(), + Admin.SnapshotType.valueOf(SnapshotDescription.Type.FLUSH.toString())); + } } // kick each off its own thread @@ -458,7 +472,7 @@ public class TestFlushSnapshotFromClient { // loop until all are done. while (true) { int doneCount = 0; - for (SnapshotDescription ss : descs) { + for (Admin.SnapshotDescription ss : descs) { try { if (admin.isSnapshotFinished(ss)) { doneCount++; @@ -477,7 +491,7 @@ public class TestFlushSnapshotFromClient { // dump for debugging UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); - List taken = admin.listSnapshots(); + List taken = admin.listSnapshots(); int takenSize = taken.size(); LOG.info("Taken " + takenSize + " snapshots: " + taken); assertTrue("We expect at least 1 request to be rejected because of we concurrently" + @@ -486,7 +500,7 @@ public class TestFlushSnapshotFromClient { // Verify that there's at least one snapshot per table int t1SnapshotsCount = 0; int t2SnapshotsCount = 0; - for (SnapshotDescription ss : taken) { + for (Admin.SnapshotDescription ss : taken) { if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) { t1SnapshotsCount++; } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java index 877ee21..51a1a2f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Admin.SnapshotType; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; @@ -116,7 +117,7 @@ public class TestRestoreFlushSnapshotFromClient { // take a snapshot admin.snapshot(Bytes.toString(snapshotName0), tableName, - SnapshotDescription.Type.FLUSH); + SnapshotType.valueOf(SnapshotDescription.Type.FLUSH.toString())); LOG.info("=== after snapshot with 500 rows"); logFSTree(); @@ -129,7 +130,7 @@ public class TestRestoreFlushSnapshotFromClient { // take a snapshot of the updated table admin.snapshot(Bytes.toString(snapshotName1), tableName, - SnapshotDescription.Type.FLUSH); + SnapshotType.valueOf(SnapshotDescription.Type.FLUSH.toString())); LOG.info("=== after snapshot with 1000 rows"); logFSTree(); table.close(); @@ -194,7 +195,8 @@ public class TestRestoreFlushSnapshotFromClient { TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis()); admin.cloneSnapshot(snapshotName0, clonedTableName); verifyRowCount(UTIL, clonedTableName, snapshot0Rows); - admin.snapshot(Bytes.toString(snapshotName2), clonedTableName, SnapshotDescription.Type.FLUSH); + admin.snapshot(Bytes.toString(snapshotName2), clonedTableName, + SnapshotType.valueOf(SnapshotDescription.Type.FLUSH.toString())); UTIL.deleteTable(clonedTableName); admin.cloneSnapshot(snapshotName2, clonedTableName);