diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupClientUtil.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupClientUtil.java deleted file mode 100644 index 072adaf..0000000 --- hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupClientUtil.java +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -/** - * A collection of methods used by multiple classes to backup HBase tables. - */ -@InterfaceAudience.Public -@InterfaceStability.Evolving -public final class BackupClientUtil { - protected static final Log LOG = LogFactory.getLog(BackupClientUtil.class); - public static final String LOGNAME_SEPARATOR = "."; - - private BackupClientUtil(){ - throw new AssertionError("Instantiating utility class..."); - } - - /** - * Check whether the backup path exist - * @param backupStr backup - * @param conf configuration - * @return Yes if path exists - * @throws IOException exception - */ - public static boolean checkPathExist(String backupStr, Configuration conf) - throws IOException { - boolean isExist = false; - Path backupPath = new Path(backupStr); - FileSystem fileSys = backupPath.getFileSystem(conf); - String targetFsScheme = fileSys.getUri().getScheme(); - if (LOG.isTraceEnabled()) { - LOG.trace("Schema of given url: " + backupStr + " is: " + targetFsScheme); - } - if (fileSys.exists(backupPath)) { - isExist = true; - } - return isExist; - } - - // check target path first, confirm it doesn't exist before backup - public static void checkTargetDir(String backupRootPath, Configuration conf) throws IOException { - boolean targetExists = false; - try { - targetExists = checkPathExist(backupRootPath, conf); - } catch (IOException e) { - String expMsg = e.getMessage(); - String newMsg = null; - if (expMsg.contains("No FileSystem for scheme")) { - newMsg = - "Unsupported filesystem scheme found in the backup target url. Error Message: " - + newMsg; - LOG.error(newMsg); - throw new IOException(newMsg); - } else { - throw e; - } - } catch (RuntimeException e) { - LOG.error(e.getMessage()); - throw e; - } - - if (targetExists) { - LOG.info("Using existing backup root dir: " + backupRootPath); - } else { - LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created."); - } - } - - /** - * Get the min value for all the Values a map. - * @param map map - * @return the min value - */ - public static Long getMinValue(HashMap map) { - Long minTimestamp = null; - if (map != null) { - ArrayList timestampList = new ArrayList(map.values()); - Collections.sort(timestampList); - // The min among all the RS log timestamps will be kept in hbase:backup table. - minTimestamp = timestampList.get(0); - } - return minTimestamp; - } - - /** - * TODO: verify the code - * @param p path - * @return host name - * @throws IOException exception - */ - public static String parseHostFromOldLog(Path p) throws IOException { - String n = p.getName(); - int idx = n.lastIndexOf(LOGNAME_SEPARATOR); - String s = URLDecoder.decode(n.substring(0, idx), "UTF8"); - return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s); - } - - /** - * Given the log file, parse the timestamp from the file name. The timestamp is the last number. - * @param p a path to the log file - * @return the timestamp - * @throws IOException exception - */ - public static Long getCreationTime(Path p) throws IOException { - int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR); - if (idx < 0) { - throw new IOException("Cannot parse timestamp from path " + p); - } - String ts = p.getName().substring(idx + 1); - return Long.parseLong(ts); - } - - public static List getFiles(FileSystem fs, Path rootDir, List files, - PathFilter filter) throws FileNotFoundException, IOException { - RemoteIterator it = fs.listFiles(rootDir, true); - - while (it.hasNext()) { - LocatedFileStatus lfs = it.next(); - if (lfs.isDirectory()) { - continue; - } - // apply filter - if (filter.accept(lfs.getPath())) { - files.add(lfs.getPath().toString()); - } - } - return files; - } -} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java new file mode 100644 index 0000000..cd299c5 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java @@ -0,0 +1,485 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupType; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; +import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.BackupInfo.Builder; +import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.TableBackupStatus; + + +/** + * An object to encapsulate the information for each backup request + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class BackupInfo implements Comparable { + private static final Log LOG = LogFactory.getLog(BackupInfo.class); + // backup status flag + public static enum BackupState { + WAITING, RUNNING, COMPLETE, FAILED, CANCELLED, ANY; + } + // backup phase + public static enum BackupPhase { + SNAPSHOTCOPY, INCREMENTAL_COPY, STORE_MANIFEST; + } + + // backup id: a timestamp when we request the backup + private String backupId; + + // backup type, full or incremental + private BackupType type; + + // target root directory for storing the backup files + private String targetRootDir; + + // overall backup state + private BackupState state; + + // overall backup phase + private BackupPhase phase; + + // overall backup failure message + private String failedMsg; + + // backup status map for all tables + private Map backupStatusMap; + + // actual start timestamp of the backup process + private long startTs; + + // actual end timestamp of the backup process, could be fail or complete + private long endTs; + + // the total bytes of incremental logs copied + private long totalBytesCopied; + + // for incremental backup, the location of the backed-up hlogs + private String hlogTargetDir = null; + + // incremental backup file list + transient private List incrBackupFileList; + + // new region server log timestamps for table set after distributed log roll + // key - table name, value - map of RegionServer hostname -> last log rolled timestamp + transient private HashMap> tableSetTimestampMap; + + // backup progress in %% (0-100) + private int progress; + + // distributed job id + private String jobId; + + // Number of parallel workers. -1 - system defined + private int workers = -1; + + // Bandwidth per worker in MB per sec. -1 - unlimited + private long bandwidth = -1; + + public BackupInfo() { + } + + public BackupInfo(String backupId, BackupType type, TableName[] tables, String targetRootDir) { + backupStatusMap = new HashMap(); + + this.backupId = backupId; + this.type = type; + this.targetRootDir = targetRootDir; + if(LOG.isDebugEnabled()){ + LOG.debug("CreateBackupContext: " + tables.length+" "+tables[0] ); + } + this.addTables(tables); + + if (type == BackupType.INCREMENTAL) { + setHlogTargetDir(BackupClientUtil.getLogBackupDir(targetRootDir, backupId)); + } + + this.startTs = 0; + this.endTs = 0; + } + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public int getWorkers() { + return workers; + } + + public void setWorkers(int workers) { + this.workers = workers; + } + + public long getBandwidth() { + return bandwidth; + } + + public void setBandwidth(long bandwidth) { + this.bandwidth = bandwidth; + } + + public void setBackupStatusMap(Map backupStatusMap) { + this.backupStatusMap = backupStatusMap; + } + + public HashMap> getTableSetTimestampMap() { + return tableSetTimestampMap; + } + + public void setTableSetTimestampMap(HashMap> tableSetTimestampMap) { + this.tableSetTimestampMap = tableSetTimestampMap; + } + + public String getHlogTargetDir() { + return hlogTargetDir; + } + + public void setType(BackupType type) { + this.type = type; + } + + public void setTargetRootDir(String targetRootDir) { + this.targetRootDir = targetRootDir; + } + + public void setTotalBytesCopied(long totalBytesCopied) { + this.totalBytesCopied = totalBytesCopied; + } + + public void setCancelled(boolean cancelled) { + this.state = BackupState.CANCELLED;; + } + + /** + * Set progress (0-100%) + * @param msg progress value + */ + + public void setProgress(int p) { + this.progress = p; + } + + /** + * Get current progress + */ + public int getProgress() { + return progress; + } + + + /** + * Has been marked as cancelled or not. + * @return True if marked as cancelled + */ + public boolean isCancelled() { + return this.state == BackupState.CANCELLED; + } + + public String getBackupId() { + return backupId; + } + + public void setBackupId(String backupId) { + this.backupId = backupId; + } + + public BackupStatus getBackupStatus(TableName table) { + return this.backupStatusMap.get(table); + } + + public String getFailedMsg() { + return failedMsg; + } + + public void setFailedMsg(String failedMsg) { + this.failedMsg = failedMsg; + } + + public long getStartTs() { + return startTs; + } + + public void setStartTs(long startTs) { + this.startTs = startTs; + } + + public long getEndTs() { + return endTs; + } + + public void setEndTs(long endTs) { + this.endTs = endTs; + } + + public long getTotalBytesCopied() { + return totalBytesCopied; + } + + public BackupState getState() { + return state; + } + + public void setState(BackupState flag) { + this.state = flag; + } + + public BackupPhase getPhase() { + return phase; + } + + public void setPhase(BackupPhase phase) { + this.phase = phase; + } + + public BackupType getType() { + return type; + } + + public void setSnapshotName(TableName table, String snapshotName) { + this.backupStatusMap.get(table).setSnapshotName(snapshotName); + } + + public String getSnapshotName(TableName table) { + return this.backupStatusMap.get(table).getSnapshotName(); + } + + public List getSnapshotNames() { + List snapshotNames = new ArrayList(); + for (BackupStatus backupStatus : this.backupStatusMap.values()) { + snapshotNames.add(backupStatus.getSnapshotName()); + } + return snapshotNames; + } + + public Set getTables() { + return this.backupStatusMap.keySet(); + } + + public List getTableNames() { + return new ArrayList(backupStatusMap.keySet()); + } + + public void addTables(TableName[] tables) { + for (TableName table : tables) { + BackupStatus backupStatus = new BackupStatus(table, this.targetRootDir, this.backupId); + this.backupStatusMap.put(table, backupStatus); + } + } + + public String getTargetRootDir() { + return targetRootDir; + } + + public void setHlogTargetDir(String hlogTagetDir) { + this.hlogTargetDir = hlogTagetDir; + } + + public String getHLogTargetDir() { + return hlogTargetDir; + } + + public List getIncrBackupFileList() { + return incrBackupFileList; + } + + public void setIncrBackupFileList(List incrBackupFileList) { + this.incrBackupFileList = incrBackupFileList; + } + + /** + * Set the new region server log timestamps after distributed log roll + * @param newTableSetTimestampMap table timestamp map + */ + public void setIncrTimestampMap(HashMap> newTableSetTimestampMap) { + this.tableSetTimestampMap = newTableSetTimestampMap; + } + + /** + * Get new region server log timestamps after distributed log roll + * @return new region server log timestamps + */ + public HashMap> getIncrTimestampMap() { + return this.tableSetTimestampMap; + } + + public TableName getTableBySnapshot(String snapshotName) { + for (Entry entry : this.backupStatusMap.entrySet()) { + if (snapshotName.equals(entry.getValue().getSnapshotName())) { + return entry.getKey(); + } + } + return null; + } + + public BackupProtos.BackupInfo toProtosBackupInfo() { + BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder(); + builder.setBackupId(getBackupId()); + setBackupStatusMap(builder); + builder.setEndTs(getEndTs()); + if (getFailedMsg() != null) { + builder.setFailedMessage(getFailedMsg()); + } + if (getState() != null) { + builder.setState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name())); + } + if (getPhase() != null) { + builder.setPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name())); + } + + builder.setProgress(getProgress()); + builder.setStartTs(getStartTs()); + builder.setTargetRootDir(getTargetRootDir()); + builder.setType(BackupProtos.BackupType.valueOf(getType().name())); + builder.setWorkersNumber(workers); + builder.setBandwidth(bandwidth); + if (jobId != null) { + builder.setJobId(jobId); + } + return builder.build(); + } + + public byte[] toByteArray() throws IOException { + return toProtosBackupInfo().toByteArray(); + } + + private void setBackupStatusMap(Builder builder) { + for (Entry entry: backupStatusMap.entrySet()) { + builder.addTableBackupStatus(entry.getValue().toProto()); + } + } + + public static BackupInfo fromByteArray(byte[] data) throws IOException { + return fromProto(BackupProtos.BackupInfo.parseFrom(data)); + } + + public static BackupInfo fromStream(final InputStream stream) throws IOException { + return fromProto(BackupProtos.BackupInfo.parseDelimitedFrom(stream)); + } + + public static BackupInfo fromProto(BackupProtos.BackupInfo proto) { + BackupInfo context = new BackupInfo(); + context.setBackupId(proto.getBackupId()); + context.setBackupStatusMap(toMap(proto.getTableBackupStatusList())); + context.setEndTs(proto.getEndTs()); + if (proto.hasFailedMessage()) { + context.setFailedMsg(proto.getFailedMessage()); + } + if (proto.hasState()) { + context.setState(BackupInfo.BackupState.valueOf(proto.getState().name())); + } + + context.setHlogTargetDir(BackupClientUtil.getLogBackupDir(proto.getTargetRootDir(), + proto.getBackupId())); + + if (proto.hasPhase()) { + context.setPhase(BackupPhase.valueOf(proto.getPhase().name())); + } + if (proto.hasProgress()) { + context.setProgress(proto.getProgress()); + } + context.setStartTs(proto.getStartTs()); + context.setTargetRootDir(proto.getTargetRootDir()); + context.setType(BackupType.valueOf(proto.getType().name())); + context.setWorkers(proto.getWorkersNumber()); + context.setBandwidth(proto.getBandwidth()); + if (proto.hasJobId()) { + context.setJobId(proto.getJobId()); + } + return context; + } + + private static Map toMap(List list) { + HashMap map = new HashMap<>(); + for (TableBackupStatus tbs : list){ + map.put(ProtobufUtil.toTableName(tbs.getTable()), BackupStatus.convert(tbs)); + } + return map; + } + + public String getShortDescription() { + StringBuilder sb = new StringBuilder(); + sb.append("ID : " + backupId).append("\n"); + sb.append("Type : " + getType()).append("\n"); + sb.append("Tables : " + getTableListAsString()).append("\n"); + sb.append("State : " + getState()).append("\n"); + Date date = null; + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(getStartTs()); + date = cal.getTime(); + sb.append("Start time : " + date).append("\n"); + if (state == BackupState.FAILED) { + sb.append("Failed message : " + getFailedMsg()).append("\n"); + } else if (state == BackupState.RUNNING) { + sb.append("Phase : " + getPhase()).append("\n"); + } else if (state == BackupState.COMPLETE) { + cal = Calendar.getInstance(); + cal.setTimeInMillis(getEndTs()); + date = cal.getTime(); + sb.append("End time : " + date).append("\n"); + } + sb.append("Progress : " + getProgress()).append("\n"); + return sb.toString(); + } + + public String getStatusAndProgressAsString() { + StringBuilder sb = new StringBuilder(); + sb.append("id: ").append(getBackupId()).append(" state: ").append(getState()) + .append(" progress: ").append(getProgress()); + return sb.toString(); + } + + public String getTableListAsString() { + return StringUtils.join(backupStatusMap.keySet(), ";"); + } + + @Override + public int compareTo(BackupInfo o) { + Long thisTS = + new Long(this.getBackupId().substring(this.getBackupId().lastIndexOf("_") + 1)); + Long otherTS = + new Long(o.getBackupId().substring(o.getBackupId().lastIndexOf("_") + 1)); + return thisTS.compareTo(otherTS); + } + + +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreClientFactory.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreClientFactory.java new file mode 100644 index 0000000..b60ab21 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreClientFactory.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.util.ReflectionUtils; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class BackupRestoreClientFactory { + private static final Log LOG = LogFactory.getLog(BackupRestoreClientFactory.class); + + private BackupRestoreClientFactory(){ + throw new AssertionError("Instantiating utility class..."); + } + + + /** + * Gets restore client implementation + * @param conf - configuration + * @return backup client + */ + public static RestoreClient getRestoreClient(Configuration conf) { + try{ + Class cls = + conf.getClassByName("org.apache.hadoop.hbase.backup.impl.RestoreClientImpl"); + + RestoreClient client = (RestoreClient) ReflectionUtils.newInstance(cls, conf); + client.setConf(conf); + return client; + } catch(Exception e){ + LOG.error("Can not instantiate RestoreClient. Make sure you have hbase-server jar in CLASSPATH", e); + } + return null; + } +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java new file mode 100644 index 0000000..c82e05a --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup; + +import java.io.Serializable; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; + +/** + * Backup status and related information encapsulated for a table. + * At this moment only TargetDir and SnapshotName is encapsulated here. + */ + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class BackupStatus implements Serializable { + + private static final long serialVersionUID = -5968397963548535982L; + + // table name for backup + private TableName table; + + // target directory of the backup image for this table + private String targetDir; + + // snapshot name for offline/online snapshot + private String snapshotName = null; + + public BackupStatus() { + + } + + public BackupStatus(TableName table, String targetRootDir, String backupId) { + this.table = table; + this.targetDir = BackupClientUtil.getTableBackupDir(targetRootDir, backupId, table); + } + + public String getSnapshotName() { + return snapshotName; + } + + public void setSnapshotName(String snapshotName) { + this.snapshotName = snapshotName; + } + + public String getTargetDir() { + return targetDir; + } + + public TableName getTable() { + return table; + } + + public void setTable(TableName table) { + this.table = table; + } + + public void setTargetDir(String targetDir) { + this.targetDir = targetDir; + } + + public static BackupStatus convert(BackupProtos.TableBackupStatus proto) + { + BackupStatus bs = new BackupStatus(); + bs.setTable(ProtobufUtil.toTableName(proto.getTable())); + bs.setTargetDir(proto.getTargetDir()); + if(proto.hasSnapshot()){ + bs.setSnapshotName(proto.getSnapshot()); + } + return bs; + } + + public BackupProtos.TableBackupStatus toProto() { + BackupProtos.TableBackupStatus.Builder builder = + BackupProtos.TableBackupStatus.newBuilder(); + if(snapshotName != null) { + builder.setSnapshot(snapshotName); + } + builder.setTable(ProtobufUtil.toProtoTableName(table)); + builder.setTargetDir(targetDir); + return builder.build(); + } +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java new file mode 100644 index 0000000..acbcb44 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface RestoreClient { + + public void setConf(Configuration conf); + + /** + * Restore operation. + * @param backupRootDir The root dir for backup image + * @param backupId The backup id for image to be restored + * @param check True if only do dependency check + * @param autoRestore True if automatically restore following the dependency + * @param sTableArray The array of tables to be restored + * @param tTableArray The array of mapping tables to restore to + * @param isOverwrite True then do restore overwrite if target table exists, otherwise fail the + * request if target table exists + * @throws IOException if any failure during restore + */ + public void restore( + String backupRootDir, + String backupId, boolean check, boolean autoRestore, TableName[] sTableArray, + TableName[] tTableArray, boolean isOverwrite) throws IOException; +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java new file mode 100644 index 0000000..770e588 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Evolving + +/** + * POJO class for restore request + */ +public class RestoreRequest { + + private String backupRootDir; + private String backupId; + private boolean check = false; + private boolean autorestore = true; + private TableName[] fromTables; + private TableName[] toTables; + private boolean overwrite = false; + + public RestoreRequest(){ + } + + public String getBackupRootDir() { + return backupRootDir; + } + + public RestoreRequest setBackupRootDir(String backupRootDir) { + this.backupRootDir = backupRootDir; + return this; + } + + public String getBackupId() { + return backupId; + } + + public RestoreRequest setBackupId(String backupId) { + this.backupId = backupId; + return this; + } + + public boolean isCheck() { + return check; + } + + public RestoreRequest setCheck(boolean check) { + this.check = check; + return this; + } + + public boolean isAutorestore() { + return autorestore; + } + + public RestoreRequest setAutorestore(boolean autorestore) { + this.autorestore = autorestore; + return this; + } + + public TableName[] getFromTables() { + return fromTables; + } + + public RestoreRequest setFromTables(TableName[] fromTables) { + this.fromTables = fromTables; + return this; + + } + + public TableName[] getToTables() { + return toTables; + } + + public RestoreRequest setToTables(TableName[] toTables) { + this.toTables = toTables; + return this; + } + + public boolean isOverwrite() { + return overwrite; + } + + public RestoreRequest setOverwrite(boolean overwrite) { + this.overwrite = overwrite; + return this; + } + + +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java new file mode 100644 index 0000000..b936840 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -0,0 +1,551 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.impl; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupRequest; +import org.apache.hadoop.hbase.backup.BackupType; +import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants.BackupCommand; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.backup.util.BackupSet; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.BackupAdmin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; + +import com.google.common.collect.Lists; + +/** + * General backup commands, options and usage messages + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class BackupCommands { + + private static final String USAGE = "Usage: hbase backup COMMAND\n" + + "where COMMAND is one of:\n" + + " create create a new backup image\n" + + " cancel cancel an ongoing backup\n" + + " delete delete an existing backup image\n" + + " describe show the detailed information of a backup image\n" + + " history show history of all successful backups\n" + + " progress show the progress of the latest backup request\n" + + " set backup set management\n" + + "Enter \'help COMMAND\' to see help message for each command\n"; + + private static final String CREATE_CMD_USAGE = + "Usage: hbase backup create [tables] [-s name] [-convert] " + + "[-silent] [-w workers][-b bandwith]\n" + " type \"full\" to create a full backup image;\n" + + " \"incremental\" to create an incremental backup image\n" + + " backup_root_path The full root path to store the backup image,\n" + + " the prefix can be hdfs, webhdfs or gpfs\n" + " Options:\n" + + " tables If no tables (\"\") are specified, all tables are backed up. " + + "Otherwise it is a\n" + " comma separated list of tables.\n" + + " -w number of parallel workers.\n" + + " -b bandwith per one worker (in MB sec)\n" + + " -set name of backup set" ; + + private static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress \n" + + " backupId backup image id;\n"; + + private static final String DESCRIBE_CMD_USAGE = "Usage: hbase backup decsribe \n" + + " backupId backup image id\n"; + + private static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [-n N]\n" + + " -n N show up to N last backup sessions, default - 10;\n"; + + private static final String DELETE_CMD_USAGE = "Usage: hbase backup delete \n" + + " backupId backup image id;\n"; + + private static final String CANCEL_CMD_USAGE = "Usage: hbase backup cancel \n" + + " backupId backup image id;\n"; + + private static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n" + + " name Backup set name\n" + + " tables If no tables (\"\") are specified, all tables will belong to the set. " + + "Otherwise it is a\n" + " comma separated list of tables.\n" + + "where COMMAND is one of:\n" + + " add add tables to a set, crete set if needed\n" + + " remove remove tables from set\n" + + " list list all sets\n" + + " describe describes set\n" + + " delete delete backup set\n"; + + public static abstract class Command extends Configured { + Command(Configuration conf) { + super(conf); + } + public abstract void execute() throws IOException; + } + + private BackupCommands() { + throw new AssertionError("Instantiating utility class..."); + } + + public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) { + Command cmd = null; + switch (type) { + case CREATE: + cmd = new CreateCommand(conf, cmdline); + break; + case DESCRIBE: + cmd = new DescribeCommand(conf, cmdline); + break; + case PROGRESS: + cmd = new ProgressCommand(conf, cmdline); + break; + case DELETE: + cmd = new DeleteCommand(conf, cmdline); + break; + case CANCEL: + cmd = new CancelCommand(conf, cmdline); + break; + case HISTORY: + cmd = new HistoryCommand(conf, cmdline); + break; + case SET: + cmd = new BackupSetCommand(conf, cmdline); + break; + case HELP: + default: + cmd = new HelpCommand(conf, cmdline); + break; + } + return cmd; + } + + + public static class CreateCommand extends Command { + CommandLine cmdline; + + CreateCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + if (cmdline == null || cmdline.getArgs() == null) { + System.out.println("ERROR: missing arguments"); + System.out.println(CREATE_CMD_USAGE); + System.exit(-1); + } + String[] args = cmdline.getArgs(); + if (args.length < 3 || args.length > 4) { + System.out.println("ERROR: wrong number of arguments"); + System.out.println(CREATE_CMD_USAGE); + System.exit(-1); + } + + if (!BackupType.FULL.toString().equalsIgnoreCase(args[1]) + && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) { + System.out.println("ERROR: invalid backup type"); + System.out.println(CREATE_CMD_USAGE); + System.exit(-1); + } + + String tables = null; + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + + // Check backup set + if (cmdline.hasOption("set")) { + String setName = cmdline.getOptionValue("set"); + tables = getTablesForSet(setName, conf); + + if (tables == null) throw new IOException("Backup set '" + setName + + "' is either empty or does not exist"); + } else { + tables = (args.length == 4) ? args[3] : null; + } + int bandwidth = cmdline.hasOption('b') ? Integer.parseInt(cmdline.getOptionValue('b')) : -1; + int workers = cmdline.hasOption('w') ? Integer.parseInt(cmdline.getOptionValue('w')) : -1; + + try (Connection conn = ConnectionFactory.createConnection(getConf()); + Admin admin = conn.getAdmin(); + BackupAdmin backupAdmin = admin.getBackupAdmin();) { + BackupRequest request = new BackupRequest(); + request.setBackupType(BackupType.valueOf(args[1].toUpperCase())) + .setTableList(tables != null?Lists.newArrayList(BackupClientUtil.parseTableNames(tables)): null) + .setTargetRootDir(args[2]).setWorkers(workers).setBandwidth(bandwidth); + backupAdmin.backupTables(request); + } catch (IOException e) { + throw e; + } + } + private String getTablesForSet(String name, Configuration conf) + throws IOException { + try (final Connection conn = ConnectionFactory.createConnection(conf); + final BackupSystemTable table = new BackupSystemTable(conn)) { + List tables = table.describeBackupSet(name); + if (tables == null) return null; + return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); + } + } + } + + private static class HelpCommand extends Command { + CommandLine cmdline; + + HelpCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + if (cmdline == null) { + System.out.println(USAGE); + System.exit(0); + } + + String[] args = cmdline.getArgs(); + if (args == null || args.length == 0) { + System.out.println(USAGE); + System.exit(0); + } + + if (args.length != 2) { + System.out.println("Only support check help message of a single command type"); + System.out.println(USAGE); + System.exit(0); + } + + String type = args[1]; + + if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) { + System.out.println(CREATE_CMD_USAGE); + } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) { + System.out.println(DESCRIBE_CMD_USAGE); + } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) { + System.out.println(HISTORY_CMD_USAGE); + } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) { + System.out.println(PROGRESS_CMD_USAGE); + } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) { + System.out.println(DELETE_CMD_USAGE); + } else if (BackupCommand.CANCEL.name().equalsIgnoreCase(type)) { + System.out.println(CANCEL_CMD_USAGE); + } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) { + System.out.println(SET_CMD_USAGE); + } else { + System.out.println("Unknown command : " + type); + System.out.println(USAGE); + } + System.exit(0); + } + } + + private static class DescribeCommand extends Command { + CommandLine cmdline; + + DescribeCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + if (cmdline == null || cmdline.getArgs() == null) { + System.out.println("ERROR: missing arguments"); + System.out.println(DESCRIBE_CMD_USAGE); + System.exit(-1); + } + String[] args = cmdline.getArgs(); + if (args.length != 2) { + System.out.println("ERROR: wrong number of arguments"); + System.out.println(DESCRIBE_CMD_USAGE); + System.exit(-1); + } + + String backupId = args[1]; + Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); + try (final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) { + BackupInfo info = admin.getBackupInfo(backupId); + System.out.println(info.getShortDescription()); + } + } + } + + private static class ProgressCommand extends Command { + CommandLine cmdline; + + ProgressCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + if (cmdline == null || cmdline.getArgs() == null || + cmdline.getArgs().length != 2) { + System.out.println("No backup id was specified, " + + "will retrieve the most recent (ongoing) sessions"); + } + String[] args = cmdline.getArgs(); + if (args.length > 2) { + System.out.println("ERROR: wrong number of arguments: " + args.length); + System.out.println(PROGRESS_CMD_USAGE); + System.exit(-1); + } + + String backupId = args == null ? null : args[1]; + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ + int progress = admin.getProgress(backupId); + if(progress < 0){ + System.out.println("No info was found for backup id: "+backupId); + } else{ + System.out.println(backupId+" progress=" + progress+"%"); + } + } + } + } + + private static class DeleteCommand extends Command { + + CommandLine cmdline; + DeleteCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { + System.out.println("No backup id(s) was specified"); + System.out.println(PROGRESS_CMD_USAGE); + System.exit(-1); + } + String[] args = cmdline.getArgs(); + + String[] backupIds = new String[args.length - 1]; + System.arraycopy(args, 1, backupIds, 0, backupIds.length); + Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); + try (final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) { + int deleted = admin.deleteBackups(args); + System.out.println("Deleted " + deleted + " backups. Total requested: " + args.length); + } + + } + } + +// TODO Cancel command + + private static class CancelCommand extends Command { + CommandLine cmdline; + + CancelCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { + System.out.println("No backup id(s) was specified, will use the most recent one"); + } + String[] args = cmdline.getArgs(); + String backupId = args == null || args.length == 0 ? null : args[1]; + Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); + try (final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) { + // TODO cancel backup + } + } + } + + private static class HistoryCommand extends Command { + CommandLine cmdline; + private final static int DEFAULT_HISTORY_LENGTH = 10; + + HistoryCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + + int n = parseHistoryLength(); + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ + List history = admin.getHistory(n); + for(BackupInfo info: history){ + System.out.println(info.getShortDescription()); + } + } + } + + private int parseHistoryLength() { + String value = cmdline.getOptionValue("n"); + if (value == null) return DEFAULT_HISTORY_LENGTH; + return Integer.parseInt(value); + } + } + + private static class BackupSetCommand extends Command { + private final static String SET_ADD_CMD = "add"; + private final static String SET_REMOVE_CMD = "remove"; + private final static String SET_DELETE_CMD = "delete"; + private final static String SET_DESCRIBE_CMD = "describe"; + private final static String SET_LIST_CMD = "list"; + + CommandLine cmdline; + + BackupSetCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + + // Command-line must have at least one element + if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { + throw new IOException("command line format"); + } + String[] args = cmdline.getArgs(); + String cmdStr = args[1]; + BackupCommand cmd = getCommand(cmdStr); + + switch (cmd) { + case SET_ADD: + processSetAdd(args); + break; + case SET_REMOVE: + processSetRemove(args); + break; + case SET_DELETE: + processSetDelete(args); + break; + case SET_DESCRIBE: + processSetDescribe(args); + break; + case SET_LIST: + processSetList(args); + break; + default: + break; + + } + } + + private void processSetList(String[] args) throws IOException { + // List all backup set names + // does not expect any args + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ + admin.listBackupSets(); + } + } + + private void processSetDescribe(String[] args) throws IOException { + if (args == null || args.length != 3) { + throw new RuntimeException("Wrong number of args: "+args.length); + } + String setName = args[2]; + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ + BackupSet set = admin.getBackupSet(setName); + System.out.println(set); + } + } + + private void processSetDelete(String[] args) throws IOException { + if (args == null || args.length != 3) { + throw new RuntimeException("Wrong number of args"); + } + String setName = args[2]; + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ + boolean result = admin.deleteBackupSet(setName); + if(result){ + System.out.println("Delete set "+setName+" OK."); + } else{ + System.out.println("Set "+setName+" does not exist"); + } + } + } + + private void processSetRemove(String[] args) throws IOException { + if (args == null || args.length != 4) { + throw new RuntimeException("Wrong args"); + } + String setName = args[2]; + String[] tables = args[3].split(","); + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ + admin.removeFromBackupSet(setName, tables); + } + } + + private void processSetAdd(String[] args) throws IOException { + if (args == null || args.length != 4) { + throw new RuntimeException("Wrong args"); + } + String setName = args[2]; + String[] tables = args[3].split(","); + TableName[] tableNames = new TableName[tables.length]; + for(int i=0; i < tables.length; i++){ + tableNames[i] = TableName.valueOf(tables[i]); + } + Configuration conf = getConf() != null? getConf():HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ + admin.addToBackupSet(setName, tableNames); + } + + } + + private BackupCommand getCommand(String cmdStr) throws IOException { + if (cmdStr.equals(SET_ADD_CMD)) { + return BackupCommand.SET_ADD; + } else if (cmdStr.equals(SET_REMOVE_CMD)) { + return BackupCommand.SET_REMOVE; + } else if (cmdStr.equals(SET_DELETE_CMD)) { + return BackupCommand.SET_DELETE; + } else if (cmdStr.equals(SET_DESCRIBE_CMD)) { + return BackupCommand.SET_DESCRIBE; + } else if (cmdStr.equals(SET_LIST_CMD)) { + return BackupCommand.SET_LIST; + } else { + throw new IOException("Unknown command for 'set' :" + cmdStr); + } + } + + } +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java new file mode 100644 index 0000000..ca204b4 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.impl; + +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * Backup exception + */ +@SuppressWarnings("serial") +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class BackupException extends HBaseIOException { + private BackupInfo description; + + /** + * Some exception happened for a backup and don't even know the backup that it was about + * @param msg Full description of the failure + */ + public BackupException(String msg) { + super(msg); + } + + /** + * Some exception happened for a backup with a cause + * @param cause the cause + */ + public BackupException(Throwable cause) { + super(cause); + } + + /** + * Exception for the given backup that has no previous root cause + * @param msg reason why the backup failed + * @param desc description of the backup that is being failed + */ + public BackupException(String msg, BackupInfo desc) { + super(msg); + this.description = desc; + } + + /** + * Exception for the given backup due to another exception + * @param msg reason why the backup failed + * @param cause root cause of the failure + * @param desc description of the backup that is being failed + */ + public BackupException(String msg, Throwable cause, BackupInfo desc) { + super(msg, cause); + this.description = desc; + } + + /** + * Exception when the description of the backup cannot be determined, due to some other root + * cause + * @param message description of what caused the failure + * @param e root cause + */ + public BackupException(String message, Exception e) { + super(message, e); + } + + public BackupInfo getBackupContext() { + return this.description; + } + +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupRestoreConstants.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupRestoreConstants.java new file mode 100644 index 0000000..ac1d2bc --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupRestoreConstants.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.impl; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * BackupRestoreConstants holds a bunch of HBase Backup and Restore constants + */ +@InterfaceAudience.Private +@InterfaceStability.Stable +public final class BackupRestoreConstants { + + + // delimiter in tablename list in restore command + public static final String TABLENAME_DELIMITER_IN_COMMAND = ","; + + public static final String CONF_STAGING_ROOT = "snapshot.export.staging.root"; + + public static final String BACKUPID_PREFIX = "backup_"; + + public static enum BackupCommand { + CREATE, CANCEL, DELETE, DESCRIBE, HISTORY, STATUS, CONVERT, MERGE, STOP, SHOW, HELP, PROGRESS, SET, + SET_ADD, SET_REMOVE, SET_DELETE, SET_DESCRIBE, SET_LIST + } + + private BackupRestoreConstants() { + // Can't be instantiated with this ctor. + } +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java new file mode 100644 index 0000000..2eb41a7 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -0,0 +1,808 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.impl; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; + +/** + * This class provides 'hbase:backup' table API + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class BackupSystemTable implements Closeable { + + static class WALItem { + String backupId; + String walFile; + String backupRoot; + + WALItem(String backupId, String walFile, String backupRoot) + { + this.backupId = backupId; + this.walFile = walFile; + this.backupRoot = backupRoot; + } + + public String getBackupId() { + return backupId; + } + + public String getWalFile() { + return walFile; + } + + public String getBackupRoot() { + return backupRoot; + } + + public String toString() { + return "/"+ backupRoot + "/"+backupId + "/" + walFile; + } + + } + + private static final Log LOG = LogFactory.getLog(BackupSystemTable.class); + private final static TableName tableName = TableName.BACKUP_TABLE_NAME; + // Stores backup sessions (contexts) + final static byte[] SESSIONS_FAMILY = "session".getBytes(); + // Stores other meta + final static byte[] META_FAMILY = "meta".getBytes(); + // Connection to HBase cluster, shared + // among all instances + private final Connection connection; + + public BackupSystemTable(Connection conn) throws IOException { + this.connection = conn; + } + + + public void close() { + // do nothing + } + + /** + * Updates status (state) of a backup session in hbase:backup table + * @param context context + * @throws IOException exception + */ + public void updateBackupInfo(BackupInfo context) throws IOException { + + if (LOG.isDebugEnabled()) { + LOG.debug("update backup status in hbase:backup for: " + context.getBackupId() + + " set status=" + context.getState()); + } + try (Table table = connection.getTable(tableName)) { + Put put = BackupSystemTableHelper.createPutForBackupContext(context); + table.put(put); + } + } + + /** + * Deletes backup status from hbase:backup table + * @param backupId backup id + * @return true, if operation succeeded, false - otherwise + * @throws IOException exception + */ + + public void deleteBackupInfo(String backupId) throws IOException { + + if (LOG.isDebugEnabled()) { + LOG.debug("delete backup status in hbase:backup for " + backupId); + } + try (Table table = connection.getTable(tableName)) { + Delete del = BackupSystemTableHelper.createDeleteForBackupInfo(backupId); + table.delete(del); + } + } + + /** + * Reads backup status object (instance of BackupContext) from hbase:backup table + * @param backupId - backupId + * @return Current status of backup session or null + */ + + public BackupInfo readBackupInfo(String backupId) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("read backup status from hbase:backup for: " + backupId); + } + + try (Table table = connection.getTable(tableName)) { + Get get = BackupSystemTableHelper.createGetForBackupContext(backupId); + Result res = table.get(get); + if(res.isEmpty()){ + return null; + } + return BackupSystemTableHelper.resultToBackupInfo(res); + } + } + + /** + * Read the last backup start code (timestamp) of last successful backup. Will return null if + * there is no start code stored on hbase or the value is of length 0. These two cases indicate + * there is no successful backup completed so far. + * @param backupRoot root directory path to backup + * @return the timestamp of last successful backup + * @throws IOException exception + */ + public String readBackupStartCode(String backupRoot) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("read backup start code from hbase:backup"); + } + try (Table table = connection.getTable(tableName)) { + Get get = BackupSystemTableHelper.createGetForStartCode(backupRoot); + Result res = table.get(get); + if (res.isEmpty()) { + return null; + } + Cell cell = res.listCells().get(0); + byte[] val = CellUtil.cloneValue(cell); + if (val.length == 0){ + return null; + } + return new String(val); + } + } + + /** + * Write the start code (timestamp) to hbase:backup. If passed in null, then write 0 byte. + * @param startCode start code + * @param backupRoot root directory path to backup + * @throws IOException exception + */ + public void writeBackupStartCode(Long startCode, String backupRoot) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("write backup start code to hbase:backup " + startCode); + } + try (Table table = connection.getTable(tableName)) { + Put put = BackupSystemTableHelper.createPutForStartCode(startCode.toString(), backupRoot); + table.put(put); + } + } + + /** + * Get the Region Servers log information after the last log roll from hbase:backup. + * @param backupRoot root directory path to backup + * @return RS log info + * @throws IOException exception + */ + public HashMap readRegionServerLastLogRollResult(String backupRoot) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("read region server last roll log result to hbase:backup"); + } + + Scan scan = BackupSystemTableHelper.createScanForReadRegionServerLastLogRollResult(backupRoot); + scan.setMaxVersions(1); + + try (Table table = connection.getTable(tableName); + ResultScanner scanner = table.getScanner(scan)) { + Result res = null; + HashMap rsTimestampMap = new HashMap(); + while ((res = scanner.next()) != null) { + res.advance(); + Cell cell = res.current(); + byte[] row = CellUtil.cloneRow(cell); + String server = + BackupSystemTableHelper.getServerNameForReadRegionServerLastLogRollResult(row); + byte[] data = CellUtil.cloneValue(cell); + rsTimestampMap.put(server, Long.parseLong(new String(data))); + } + return rsTimestampMap; + } + } + + /** + * Writes Region Server last roll log result (timestamp) to hbase:backup table + * @param server - Region Server name + * @param timestamp - last log timestamp + * @param backupRoot root directory path to backup + * @throws IOException exception + */ + public void writeRegionServerLastLogRollResult(String server, Long ts, String backupRoot) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("write region server last roll log result to hbase:backup"); + } + try (Table table = connection.getTable(tableName)) { + Put put = + BackupSystemTableHelper.createPutForRegionServerLastLogRollResult(server,ts,backupRoot); + table.put(put); + } + } + + /** + * Get all completed backup information (in desc order by time) + * @param onlyCompeleted, true, if only successfully completed sessions + * @return history info of BackupCompleteData + * @throws IOException exception + */ + public ArrayList getBackupHistory(boolean onlyCompleted) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("get backup history from hbase:backup"); + } + ArrayList list ; + BackupState state = onlyCompleted? BackupState.COMPLETE: BackupState.ANY; + list = getBackupContexts(state); + return BackupClientUtil.sortHistoryListDesc(list); + } + + public ArrayList getBackupHistory() throws IOException { + return getBackupHistory(false); + } + + /** + * Get all backup session with a given status (in desc order by time) + * @param status status + * @return history info of backup contexts + * @throws IOException exception + */ + public ArrayList getBackupContexts(BackupState status) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("get backup contexts from hbase:backup"); + } + + Scan scan = BackupSystemTableHelper.createScanForBackupHistory(); + ArrayList list = new ArrayList(); + + try (Table table = connection.getTable(tableName); + ResultScanner scanner = table.getScanner(scan)) { + Result res = null; + while ((res = scanner.next()) != null) { + res.advance(); + BackupInfo context = BackupSystemTableHelper.cellToBackupInfo(res.current()); + if (status != BackupState.ANY && context.getState() != status){ + continue; + } + list.add(context); + } + return list; + } + } + + /** + * Write the current timestamps for each regionserver to hbase:backup + * after a successful full or incremental backup. The saved timestamp is of the last + * log file that was backed up already. + * @param tables tables + * @param newTimestamps timestamps + * @param backupRoot root directory path to backup + * @throws IOException exception + */ + public void writeRegionServerLogTimestamp(Set tables, + HashMap newTimestamps, String backupRoot) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("write RS log time stamps to hbase:backup for tables ["+ + StringUtils.join(tables, ",")+"]"); + } + List puts = new ArrayList(); + for (TableName table : tables) { + byte[] smapData = toTableServerTimestampProto(table, newTimestamps).toByteArray(); + Put put = + BackupSystemTableHelper.createPutForWriteRegionServerLogTimestamp(table, + smapData, backupRoot); + puts.add(put); + } + try (Table table = connection.getTable(tableName)) { + table.put(puts); + } + } + + /** + * Read the timestamp for each region server log after the last successful backup. Each table has + * its own set of the timestamps. The info is stored for each table as a concatenated string of + * rs->timestapmp + * @param backupRoot root directory path to backup + * @return the timestamp for each region server. key: tableName value: + * RegionServer,PreviousTimeStamp + * @throws IOException exception + */ + public HashMap> readLogTimestampMap(String backupRoot) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("read RS log ts from hbase:backup for root="+ backupRoot); + } + + HashMap> tableTimestampMap = + new HashMap>(); + + Scan scan = BackupSystemTableHelper.createScanForReadLogTimestampMap(backupRoot); + try (Table table = connection.getTable(tableName); + ResultScanner scanner = table.getScanner(scan)) { + Result res = null; + while ((res = scanner.next()) != null) { + res.advance(); + Cell cell = res.current(); + byte[] row = CellUtil.cloneRow(cell); + String tabName = BackupSystemTableHelper.getTableNameForReadLogTimestampMap(row); + TableName tn = TableName.valueOf(tabName); + byte[] data = CellUtil.cloneValue(cell); + if (data == null) { + throw new IOException("Data of last backup data from hbase:backup " + + "is empty. Create a backup first."); + } + if (data != null && data.length > 0) { + HashMap lastBackup = + fromTableServerTimestampProto(BackupProtos.TableServerTimestamp.parseFrom(data)); + tableTimestampMap.put(tn, lastBackup); + } + } + return tableTimestampMap; + } + } + + private BackupProtos.TableServerTimestamp toTableServerTimestampProto(TableName table, + Map map) { + BackupProtos.TableServerTimestamp.Builder tstBuilder = + BackupProtos.TableServerTimestamp.newBuilder(); + tstBuilder.setTable(ProtobufUtil.toProtoTableName(table)); + + for(Entry entry: map.entrySet()) { + BackupProtos.ServerTimestamp.Builder builder = BackupProtos.ServerTimestamp.newBuilder(); + builder.setServer(entry.getKey()); + builder.setTimestamp(entry.getValue()); + tstBuilder.addServerTimestamp(builder.build()); + } + + return tstBuilder.build(); + } + + private HashMap fromTableServerTimestampProto( + BackupProtos.TableServerTimestamp proto) { + HashMap map = new HashMap (); + List list = proto.getServerTimestampList(); + for(BackupProtos.ServerTimestamp st: list) { + map.put(st.getServer(), st.getTimestamp()); + } + return map; + } + + /** + * Return the current tables covered by incremental backup. + * @param backupRoot root directory path to backup + * @return set of tableNames + * @throws IOException exception + */ + public Set getIncrementalBackupTableSet(String backupRoot) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("get incr backup table set from hbase:backup"); + } + TreeSet set = new TreeSet<>(); + + try (Table table = connection.getTable(tableName)) { + Get get = BackupSystemTableHelper.createGetForIncrBackupTableSet(backupRoot); + Result res = table.get(get); + if (res.isEmpty()) { + return set; + } + List cells = res.listCells(); + for (Cell cell : cells) { + // qualifier = table name - we use table names as qualifiers + set.add(TableName.valueOf(CellUtil.cloneQualifier(cell))); + } + return set; + } + } + + /** + * Add tables to global incremental backup set + * @param tables - set of tables + * @param backupRoot root directory path to backup + * @throws IOException exception + */ + public void addIncrementalBackupTableSet(Set tables, String backupRoot) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Add incremental backup table set to hbase:backup. ROOT="+backupRoot + + " tables ["+ StringUtils.join(tables, " ")+"]"); + for (TableName table : tables) { + LOG.debug(table); + } + } + try (Table table = connection.getTable(tableName)) { + Put put = BackupSystemTableHelper.createPutForIncrBackupTableSet(tables, backupRoot); + table.put(put); + } + } + + /** + * Register WAL files as eligible for deletion + * @param files files + * @param backupId backup id + * @param backupRoot root directory path to backup + * @throws IOException exception + */ + public void addWALFiles(List files, String backupId, + String backupRoot) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("add WAL files to hbase:backup: "+backupId +" "+backupRoot+" files ["+ + StringUtils.join(files, ",")+"]"); + for(String f: files){ + LOG.debug("add :"+f); + } + } + try (Table table = connection.getTable(tableName)) { + List puts = + BackupSystemTableHelper.createPutsForAddWALFiles(files, backupId, backupRoot); + table.put(puts); + } + } + + /** + * Register WAL files as eligible for deletion + * @param backupRoot root directory path to backup + * @throws IOException exception + */ + public Iterator getWALFilesIterator(String backupRoot) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("get WAL files from hbase:backup"); + } + final Table table = connection.getTable(tableName); + Scan scan = BackupSystemTableHelper.createScanForGetWALs(backupRoot); + final ResultScanner scanner = table.getScanner(scan); + final Iterator it = scanner.iterator(); + return new Iterator() { + + @Override + public boolean hasNext() { + boolean next = it.hasNext(); + if (!next) { + // close all + try { + scanner.close(); + table.close(); + } catch (IOException e) { + LOG.error("Close WAL Iterator", e); + } + } + return next; + } + + @Override + public WALItem next() { + Result next = it.next(); + List cells = next.listCells(); + byte[] buf = cells.get(0).getValueArray(); + int len = cells.get(0).getValueLength(); + int offset = cells.get(0).getValueOffset(); + String backupId = new String(buf, offset, len); + buf = cells.get(1).getValueArray(); + len = cells.get(1).getValueLength(); + offset = cells.get(1).getValueOffset(); + String walFile = new String(buf, offset, len); + buf = cells.get(2).getValueArray(); + len = cells.get(2).getValueLength(); + offset = cells.get(2).getValueOffset(); + String backupRoot = new String(buf, offset, len); + return new WALItem(backupId, walFile, backupRoot); + } + + @Override + public void remove() { + // not implemented + throw new RuntimeException("remove is not supported"); + } + }; + + } + + /** + * Check if WAL file is eligible for deletion + * Future: to support all backup destinations + * @param file file + * @return true, if - yes. + * @throws IOException exception + */ + public boolean isWALFileDeletable(String file) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Check if WAL file has been already backed up in hbase:backup "+ file); + } + try (Table table = connection.getTable(tableName)) { + Get get = BackupSystemTableHelper.createGetForCheckWALFile(file); + Result res = table.get(get); + if (res.isEmpty()){ + return false; + } + return true; + } + } + + /** + * Checks if we have at least one backup session in hbase:backup This API is used by + * BackupLogCleaner + * @return true, if - at least one session exists in hbase:backup table + * @throws IOException exception + */ + public boolean hasBackupSessions() throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Has backup sessions from hbase:backup"); + } + boolean result = false; + Scan scan = BackupSystemTableHelper.createScanForBackupHistory(); + scan.setCaching(1); + try (Table table = connection.getTable(tableName); + ResultScanner scanner = table.getScanner(scan)) { + if (scanner.next() != null) { + result = true; + } + return result; + } + } + + /** + * BACKUP SETS + */ + + /** + * Get backup set list + * @return backup set list + * @throws IOException + */ + public List listBackupSets() throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(" Backup set list"); + } + List list = new ArrayList(); + Table table = null; + ResultScanner scanner = null; + try { + table = connection.getTable(tableName); + Scan scan = BackupSystemTableHelper.createScanForBackupSetList(); + scan.setMaxVersions(1); + scanner = table.getScanner(scan); + Result res = null; + while ((res = scanner.next()) != null) { + res.advance(); + list.add(BackupSystemTableHelper.cellKeyToBackupSetName(res.current())); + } + return list; + } finally { + if(scanner != null) { + scanner.close(); + } + if (table != null) { + table.close(); + } + } + } + + /** + * Get backup set description (list of tables) + * @param name - set's name + * @return list of tables in a backup set + * @throws IOException + */ + public List describeBackupSet(String name) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(" Backup set describe: "+name); + } + Table table = null; + try { + table = connection.getTable(tableName); + Get get = BackupSystemTableHelper.createGetForBackupSet(name); + Result res = table.get(get); + if(res.isEmpty()) return new ArrayList(); + res.advance(); + String[] tables = + BackupSystemTableHelper.cellValueToBackupSet(res.current()); + return toList(tables); + } finally { + if (table != null) { + table.close(); + } + } + } + + private List toList(String[] tables) + { + List list = new ArrayList(tables.length); + for(String name: tables) { + list.add(TableName.valueOf(name)); + } + return list; + } + + /** + * Add backup set (list of tables) + * @param name - set name + * @param tables - list of tables, comma-separated + * @throws IOException + */ + public void addToBackupSet(String name, String[] newTables) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Backup set add: "+name+" tables ["+ StringUtils.join(newTables, " ")+"]"); + } + Table table = null; + String[] union = null; + try { + table = connection.getTable(tableName); + Get get = BackupSystemTableHelper.createGetForBackupSet(name); + Result res = table.get(get); + if(res.isEmpty()) { + union = newTables; + } else { + res.advance(); + String[] tables = + BackupSystemTableHelper.cellValueToBackupSet(res.current()); + union = merge(tables, newTables); + } + Put put = BackupSystemTableHelper.createPutForBackupSet(name, union); + table.put(put); + } finally { + if (table != null) { + table.close(); + } + } + } + + private String[] merge(String[] tables, String[] newTables) { + List list = new ArrayList(); + // Add all from tables + for(String t: tables){ + list.add(t); + } + for(String nt: newTables){ + if(list.contains(nt)) continue; + list.add(nt); + } + String[] arr = new String[list.size()]; + list.toArray(arr); + return arr; + } + + /** + * Remove tables from backup set (list of tables) + * @param name - set name + * @param tables - list of tables, comma-separated + * @throws IOException + */ + public void removeFromBackupSet(String name, String[] toRemove) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(" Backup set remove from : " + name+" tables ["+ + StringUtils.join(toRemove, " ")+"]"); + } + Table table = null; + String[] disjoint = null; + try { + table = connection.getTable(tableName); + Get get = BackupSystemTableHelper.createGetForBackupSet(name); + Result res = table.get(get); + if (res.isEmpty()) { + LOG.warn("Backup set '"+ name+"' not found."); + return; + } else { + res.advance(); + String[] tables = BackupSystemTableHelper.cellValueToBackupSet(res.current()); + disjoint = disjoin(tables, toRemove); + } + if (disjoint.length > 0) { + Put put = BackupSystemTableHelper.createPutForBackupSet(name, disjoint); + table.put(put); + } else { + // Delete + //describeBackupSet(name); + LOG.warn("Backup set '"+ name+"' does not contain tables ["+ + StringUtils.join(toRemove, " ")+"]"); + } + } finally { + if (table != null) { + table.close(); + } + } + } + + private String[] disjoin(String[] tables, String[] toRemove) { + List list = new ArrayList(); + // Add all from tables + for (String t : tables) { + list.add(t); + } + for (String nt : toRemove) { + if (list.contains(nt)) { + list.remove(nt); + } + } + String[] arr = new String[list.size()]; + list.toArray(arr); + return arr; + } + + /** + * Delete backup set + * @param name set's name + * @throws IOException + */ + public void deleteBackupSet(String name) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(" Backup set delete: " + name); + } + Table table = null; + try { + table = connection.getTable(tableName); + Delete del = BackupSystemTableHelper.createDeleteForBackupSet(name); + table.delete(del); + } finally { + if (table != null) { + table.close(); + } + } + } + + /** + * Get backup system table descriptor + * @return descriptor + */ + public static HTableDescriptor getSystemTableDescriptor() { + HTableDescriptor tableDesc = new HTableDescriptor(tableName); + HColumnDescriptor colSessionsDesc = new HColumnDescriptor(SESSIONS_FAMILY); + colSessionsDesc.setMaxVersions(1); + // Time to keep backup sessions (secs) + Configuration config = HBaseConfiguration.create(); + int ttl = + config.getInt(HConstants.BACKUP_SYSTEM_TTL_KEY, HConstants.BACKUP_SYSTEM_TTL_DEFAULT); + colSessionsDesc.setTimeToLive(ttl); + tableDesc.addFamily(colSessionsDesc); + HColumnDescriptor colMetaDesc = new HColumnDescriptor(META_FAMILY); + //colDesc.setMaxVersions(1); + tableDesc.addFamily(colMetaDesc); + return tableDesc; + } + + public static String getTableNameAsString() { + return tableName.getNameAsString(); + } + + public static TableName getTableName() { + return tableName; + } +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java new file mode 100644 index 0000000..88bb067 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java @@ -0,0 +1,425 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.impl; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; + + +/** + * A collection for methods used by BackupSystemTable. + */ + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class BackupSystemTableHelper { + + /** + * hbase:backup schema: + * 1. Backup sessions rowkey= "session:" + backupId; value = serialized + * BackupContext + * 2. Backup start code rowkey = "startcode:" + backupRoot; value = startcode + * 3. Incremental backup set rowkey="incrbackupset:" + backupRoot; value=[list of tables] + * 4. Table-RS-timestamp map rowkey="trslm:"+ backupRoot+table_name; value = map[RS-> + * last WAL timestamp] + * 5. RS - WAL ts map rowkey="rslogts:"+backupRoot +server; value = last WAL timestamp + * 6. WALs recorded rowkey="wals:"+WAL unique file name; value = backupId and full WAL file name + */ + + private final static String BACKUP_INFO_PREFIX = "session:"; + private final static String START_CODE_ROW = "startcode:"; + private final static String INCR_BACKUP_SET = "incrbackupset:"; + private final static String TABLE_RS_LOG_MAP_PREFIX = "trslm:"; + private final static String RS_LOG_TS_PREFIX = "rslogts:"; + private final static String WALS_PREFIX = "wals:"; + private final static String SET_KEY_PREFIX = "backupset:"; + + private final static byte[] EMPTY_VALUE = new byte[] {}; + + // Safe delimiter in a string + private final static String NULL = "\u0000"; + + private BackupSystemTableHelper() { + throw new AssertionError("Instantiating utility class..."); + } + + /** + * Creates Put operation for a given backup context object + * @param context backup context + * @return put operation + * @throws IOException exception + */ + static Put createPutForBackupContext(BackupInfo context) throws IOException { + Put put = new Put(rowkey(BACKUP_INFO_PREFIX, context.getBackupId())); + put.addColumn(BackupSystemTable.SESSIONS_FAMILY, "context".getBytes(), context.toByteArray()); + return put; + } + + /** + * Creates Get operation for a given backup id + * @param backupId - backup's ID + * @return get operation + * @throws IOException exception + */ + static Get createGetForBackupContext(String backupId) throws IOException { + Get get = new Get(rowkey(BACKUP_INFO_PREFIX, backupId)); + get.addFamily(BackupSystemTable.SESSIONS_FAMILY); + get.setMaxVersions(1); + return get; + } + + /** + * Creates Delete operation for a given backup id + * @param backupId - backup's ID + * @return delete operation + * @throws IOException exception + */ + public static Delete createDeleteForBackupInfo(String backupId) { + Delete del = new Delete(rowkey(BACKUP_INFO_PREFIX, backupId)); + del.addFamily(BackupSystemTable.SESSIONS_FAMILY); + return del; + } + + /** + * Converts Result to BackupContext + * @param res - HBase result + * @return backup context instance + * @throws IOException exception + */ + static BackupInfo resultToBackupInfo(Result res) throws IOException { + res.advance(); + Cell cell = res.current(); + return cellToBackupInfo(cell); + } + + /** + * Creates Get operation to retrieve start code from hbase:backup + * @return get operation + * @throws IOException exception + */ + static Get createGetForStartCode(String rootPath) throws IOException { + Get get = new Get(rowkey(START_CODE_ROW, rootPath)); + get.addFamily(BackupSystemTable.META_FAMILY); + get.setMaxVersions(1); + return get; + } + + /** + * Creates Put operation to store start code to hbase:backup + * @return put operation + * @throws IOException exception + */ + static Put createPutForStartCode(String startCode, String rootPath) { + Put put = new Put(rowkey(START_CODE_ROW, rootPath)); + put.addColumn(BackupSystemTable.META_FAMILY, "startcode".getBytes(), startCode.getBytes()); + return put; + } + + /** + * Creates Get to retrieve incremental backup table set from hbase:backup + * @return get operation + * @throws IOException exception + */ + static Get createGetForIncrBackupTableSet(String backupRoot) throws IOException { + Get get = new Get(rowkey(INCR_BACKUP_SET, backupRoot)); + get.addFamily(BackupSystemTable.META_FAMILY); + get.setMaxVersions(1); + return get; + } + + /** + * Creates Put to store incremental backup table set + * @param tables tables + * @return put operation + */ + static Put createPutForIncrBackupTableSet(Set tables, String backupRoot) { + Put put = new Put(rowkey(INCR_BACKUP_SET, backupRoot)); + for (TableName table : tables) { + put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes(table.getNameAsString()), + EMPTY_VALUE); + } + return put; + } + + /** + * Creates Scan operation to load backup history + * @return scan operation + */ + static Scan createScanForBackupHistory() { + Scan scan = new Scan(); + byte[] startRow = BACKUP_INFO_PREFIX.getBytes(); + byte[] stopRow = Arrays.copyOf(startRow, startRow.length); + stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); + scan.setStartRow(startRow); + scan.setStopRow(stopRow); + scan.addFamily(BackupSystemTable.SESSIONS_FAMILY); + scan.setMaxVersions(1); + return scan; + } + + /** + * Converts cell to backup context instance. + * @param current - cell + * @return backup context instance + * @throws IOException exception + */ + static BackupInfo cellToBackupInfo(Cell current) throws IOException { + byte[] data = CellUtil.cloneValue(current); + return BackupInfo.fromByteArray(data); + } + + /** + * Creates Put to write RS last roll log timestamp map + * @param table - table + * @param smap - map, containing RS:ts + * @return put operation + */ + static Put createPutForWriteRegionServerLogTimestamp(TableName table, byte[] smap, + String backupRoot) { + Put put = new Put(rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot, NULL, table.getNameAsString())); + put.addColumn(BackupSystemTable.META_FAMILY, "log-roll-map".getBytes(), smap); + return put; + } + + /** + * Creates Scan to load table-> { RS -> ts} map of maps + * @return scan operation + */ + static Scan createScanForReadLogTimestampMap(String backupRoot) { + Scan scan = new Scan(); + byte[] startRow = rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot); + byte[] stopRow = Arrays.copyOf(startRow, startRow.length); + stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); + scan.setStartRow(startRow); + scan.setStopRow(stopRow); + scan.addFamily(BackupSystemTable.META_FAMILY); + + return scan; + } + + /** + * Get table name from rowkey + * @param cloneRow rowkey + * @return table name + */ + static String getTableNameForReadLogTimestampMap(byte[] cloneRow) { + String s = new String(cloneRow); + int index = s.lastIndexOf(NULL); + return s.substring(index +1); + } + + /** + * Creates Put to store RS last log result + * @param server - server name + * @param timestamp - log roll result (timestamp) + * @return put operation + */ + static Put createPutForRegionServerLastLogRollResult(String server, + Long timestamp, String backupRoot ) { + Put put = new Put(rowkey(RS_LOG_TS_PREFIX, backupRoot, NULL, server)); + put.addColumn(BackupSystemTable.META_FAMILY, "rs-log-ts".getBytes(), + timestamp.toString().getBytes()); + return put; + } + + /** + * Creates Scan operation to load last RS log roll results + * @return scan operation + */ + static Scan createScanForReadRegionServerLastLogRollResult(String backupRoot) { + Scan scan = new Scan(); + byte[] startRow = rowkey(RS_LOG_TS_PREFIX, backupRoot); + byte[] stopRow = Arrays.copyOf(startRow, startRow.length); + stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); + scan.setStartRow(startRow); + scan.setStopRow(stopRow); + scan.addFamily(BackupSystemTable.META_FAMILY); + scan.setMaxVersions(1); + + return scan; + } + + /** + * Get server's name from rowkey + * @param row - rowkey + * @return server's name + */ + static String getServerNameForReadRegionServerLastLogRollResult(byte[] row) { + String s = new String(row); + int index = s.lastIndexOf(NULL); + return s.substring(index +1); + } + + /** + * Creates put list for list of WAL files + * @param files list of WAL file paths + * @param backupId backup id + * @return put list + * @throws IOException exception + */ + public static List createPutsForAddWALFiles(List files, + String backupId, String backupRoot) + throws IOException { + + List puts = new ArrayList(); + for (String file : files) { + Put put = new Put(rowkey(WALS_PREFIX, BackupClientUtil.getUniqueWALFileNamePart(file))); + put.addColumn(BackupSystemTable.META_FAMILY, "backupId".getBytes(), backupId.getBytes()); + put.addColumn(BackupSystemTable.META_FAMILY, "file".getBytes(), file.getBytes()); + put.addColumn(BackupSystemTable.META_FAMILY, "root".getBytes(), backupRoot.getBytes()); + puts.add(put); + } + return puts; + } + + /** + * Creates Scan operation to load WALs + * TODO: support for backupRoot + * @param backupRoot - path to backup destination + * @return scan operation + */ + public static Scan createScanForGetWALs(String backupRoot) { + Scan scan = new Scan(); + byte[] startRow = WALS_PREFIX.getBytes(); + byte[] stopRow = Arrays.copyOf(startRow, startRow.length); + stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); + scan.setStartRow(startRow); + scan.setStopRow(stopRow); + scan.addFamily(BackupSystemTable.META_FAMILY); + return scan; + } + /** + * Creates Get operation for a given wal file name + * TODO: support for backup destination + * @param file file + * @return get operation + * @throws IOException exception + */ + public static Get createGetForCheckWALFile(String file) throws IOException { + Get get = new Get(rowkey(WALS_PREFIX, BackupClientUtil.getUniqueWALFileNamePart(file))); + // add backup root column + get.addFamily(BackupSystemTable.META_FAMILY); + return get; + } + + + /** + * Creates Scan operation to load backup set list + * @return scan operation + */ + static Scan createScanForBackupSetList() { + Scan scan = new Scan(); + byte[] startRow = SET_KEY_PREFIX.getBytes(); + byte[] stopRow = Arrays.copyOf(startRow, startRow.length); + stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); + scan.setStartRow(startRow); + scan.setStopRow(stopRow); + scan.addFamily(BackupSystemTable.META_FAMILY); + return scan; + } + + /** + * Creates Get operation to load backup set content + * @return get operation + */ + static Get createGetForBackupSet(String name) { + Get get = new Get(rowkey(SET_KEY_PREFIX, name)); + get.addFamily(BackupSystemTable.META_FAMILY); + return get; + } + + /** + * Creates Delete operation to delete backup set content + * @return delete operation + */ + static Delete createDeleteForBackupSet(String name) { + Delete del = new Delete(rowkey(SET_KEY_PREFIX, name)); + del.addFamily(BackupSystemTable.META_FAMILY); + return del; + } + + + /** + * Creates Put operation to update backup set content + * @return put operation + */ + static Put createPutForBackupSet(String name, String[] tables) { + Put put = new Put(rowkey(SET_KEY_PREFIX, name)); + byte[] value = convertToByteArray(tables); + put.addColumn(BackupSystemTable.META_FAMILY, "tables".getBytes(), value); + return put; + } + + private static byte[] convertToByteArray(String[] tables) { + return StringUtils.join(tables, ",").getBytes(); + } + + + /** + * Converts cell to backup set list. + * @param current - cell + * @return backup set + * @throws IOException + */ + static String[] cellValueToBackupSet(Cell current) throws IOException { + byte[] data = CellUtil.cloneValue(current); + if( data != null && data.length > 0){ + return new String(data).split(","); + } else{ + return new String[0]; + } + } + + /** + * Converts cell key to backup set name. + * @param current - cell + * @return backup set name + * @throws IOException + */ + static String cellKeyToBackupSetName(Cell current) throws IOException { + byte[] data = CellUtil.cloneRow(current); + return new String(data).substring(SET_KEY_PREFIX.length()); + } + + static byte[] rowkey(String s, String ... other){ + StringBuilder sb = new StringBuilder(s); + for(String ss: other){ + sb.append(ss); + } + return sb.toString().getBytes(); + } + +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java new file mode 100644 index 0000000..1472855 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java @@ -0,0 +1,366 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.util; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A collection of methods used by multiple classes to backup HBase tables. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class BackupClientUtil { + protected static final Log LOG = LogFactory.getLog(BackupClientUtil.class); + public static final String LOGNAME_SEPARATOR = "."; + + private BackupClientUtil(){ + throw new AssertionError("Instantiating utility class..."); + } + + /** + * Check whether the backup path exist + * @param backupStr backup + * @param conf configuration + * @return Yes if path exists + * @throws IOException exception + */ + public static boolean checkPathExist(String backupStr, Configuration conf) + throws IOException { + boolean isExist = false; + Path backupPath = new Path(backupStr); + FileSystem fileSys = backupPath.getFileSystem(conf); + String targetFsScheme = fileSys.getUri().getScheme(); + if (LOG.isTraceEnabled()) { + LOG.trace("Schema of given url: " + backupStr + " is: " + targetFsScheme); + } + if (fileSys.exists(backupPath)) { + isExist = true; + } + return isExist; + } + + // check target path first, confirm it doesn't exist before backup + public static void checkTargetDir(String backupRootPath, Configuration conf) throws IOException { + boolean targetExists = false; + try { + targetExists = checkPathExist(backupRootPath, conf); + } catch (IOException e) { + String expMsg = e.getMessage(); + String newMsg = null; + if (expMsg.contains("No FileSystem for scheme")) { + newMsg = + "Unsupported filesystem scheme found in the backup target url. Error Message: " + + newMsg; + LOG.error(newMsg); + throw new IOException(newMsg); + } else { + throw e; + } + } catch (RuntimeException e) { + LOG.error(e.getMessage()); + throw e; + } + + if (targetExists) { + LOG.info("Using existing backup root dir: " + backupRootPath); + } else { + LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created."); + } + } + + /** + * Get the min value for all the Values a map. + * @param map map + * @return the min value + */ + public static Long getMinValue(HashMap map) { + Long minTimestamp = null; + if (map != null) { + ArrayList timestampList = new ArrayList(map.values()); + Collections.sort(timestampList); + // The min among all the RS log timestamps will be kept in hbase:backup table. + minTimestamp = timestampList.get(0); + } + return minTimestamp; + } + + /** + * TODO: verify the code + * @param p path + * @return host name + * @throws IOException exception + */ + public static String parseHostFromOldLog(Path p) throws IOException { + String n = p.getName(); + int idx = n.lastIndexOf(LOGNAME_SEPARATOR); + String s = URLDecoder.decode(n.substring(0, idx), "UTF8"); + return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s); + } + + /** + * Given the log file, parse the timestamp from the file name. The timestamp is the last number. + * @param p a path to the log file + * @return the timestamp + * @throws IOException exception + */ + public static Long getCreationTime(Path p) throws IOException { + int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR); + if (idx < 0) { + throw new IOException("Cannot parse timestamp from path " + p); + } + String ts = p.getName().substring(idx + 1); + return Long.parseLong(ts); + } + + public static List getFiles(FileSystem fs, Path rootDir, List files, + PathFilter filter) throws FileNotFoundException, IOException { + RemoteIterator it = fs.listFiles(rootDir, true); + + while (it.hasNext()) { + LocatedFileStatus lfs = it.next(); + if (lfs.isDirectory()) { + continue; + } + // apply filter + if (filter.accept(lfs.getPath())) { + files.add(lfs.getPath().toString()); + } + } + return files; + } + + public static void cleanupBackupData(BackupInfo context, Configuration conf) + throws IOException + { + cleanupHLogDir(context, conf); + cleanupTargetDir(context, conf); + } + + /** + * Clean up directories which are generated when DistCp copying hlogs. + * @throws IOException + */ + private static void cleanupHLogDir(BackupInfo backupContext, Configuration conf) + throws IOException { + + String logDir = backupContext.getHLogTargetDir(); + if (logDir == null) { + LOG.warn("No log directory specified for " + backupContext.getBackupId()); + return; + } + + Path rootPath = new Path(logDir).getParent(); + FileSystem fs = FileSystem.get(rootPath.toUri(), conf); + FileStatus[] files = listStatus(fs, rootPath, null); + if (files == null) { + return; + } + for (FileStatus file : files) { + LOG.debug("Delete log files: " + file.getPath().getName()); + fs.delete(file.getPath(), true); + } + } + + /** + * Clean up the data at target directory + */ + private static void cleanupTargetDir(BackupInfo backupContext, Configuration conf) { + try { + // clean up the data at target directory + LOG.debug("Trying to cleanup up target dir : " + backupContext.getBackupId()); + String targetDir = backupContext.getTargetRootDir(); + if (targetDir == null) { + LOG.warn("No target directory specified for " + backupContext.getBackupId()); + return; + } + + FileSystem outputFs = + FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf); + + for (TableName table : backupContext.getTables()) { + Path targetDirPath = + new Path(getTableBackupDir(backupContext.getTargetRootDir(), + backupContext.getBackupId(), table)); + if (outputFs.delete(targetDirPath, true)) { + LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); + } else { + LOG.info("No data has been found in " + targetDirPath.toString() + "."); + } + + Path tableDir = targetDirPath.getParent(); + FileStatus[] backups = listStatus(outputFs, tableDir, null); + if (backups == null || backups.length == 0) { + outputFs.delete(tableDir, true); + LOG.debug(tableDir.toString() + " is empty, remove it."); + } + } + + } catch (IOException e1) { + LOG.error("Cleaning up backup data of " + backupContext.getBackupId() + " at " + + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + "."); + } + } + + /** + * Given the backup root dir, backup id and the table name, return the backup image location, + * which is also where the backup manifest file is. return value look like: + * "hdfs://backup.hbase.org:9000/user/biadmin/backup1/backup_1396650096738/default/t1_dn/" + * @param backupRootDir backup root directory + * @param backupId backup id + * @param table table name + * @return backupPath String for the particular table + */ + public static String getTableBackupDir(String backupRootDir, String backupId, + TableName tableName) { + return backupRootDir + Path.SEPARATOR+ backupId + Path.SEPARATOR + + tableName.getNamespaceAsString() + Path.SEPARATOR + + tableName.getQualifierAsString() + Path.SEPARATOR ; + } + + public static TableName[] parseTableNames(String tables) { + if (tables == null) { + return null; + } + String[] tableArray = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); + + TableName[] ret = new TableName[tableArray.length]; + for (int i = 0; i < tableArray.length; i++) { + ret[i] = TableName.valueOf(tableArray[i]); + } + return ret; + } + + /** + * Sort history list by start time in descending order. + * @param historyList history list + * @return sorted list of BackupCompleteData + */ + public static ArrayList sortHistoryListDesc( + ArrayList historyList) { + ArrayList list = new ArrayList(); + TreeMap map = new TreeMap(); + for (BackupInfo h : historyList) { + map.put(Long.toString(h.getStartTs()), h); + } + Iterator i = map.descendingKeySet().iterator(); + while (i.hasNext()) { + list.add(map.get(i.next())); + } + return list; + } + + /** + * Returns WAL file name + * @param walFileName WAL file name + * @return WAL file name + * @throws IOException exception + * @throws IllegalArgumentException exception + */ + public static String getUniqueWALFileNamePart(String walFileName) throws IOException { + return getUniqueWALFileNamePart(new Path(walFileName)); + } + + /** + * Returns WAL file name + * @param p - WAL file path + * @return WAL file name + * @throws IOException exception + */ + public static String getUniqueWALFileNamePart(Path p) throws IOException { + return p.getName(); + } + + /** + * Calls fs.listStatus() and treats FileNotFoundException as non-fatal + * This accommodates differences between hadoop versions, where hadoop 1 + * does not throw a FileNotFoundException, and return an empty FileStatus[] + * while Hadoop 2 will throw FileNotFoundException. + * + * @param fs file system + * @param dir directory + * @param filter path filter + * @return null if dir is empty or doesn't exist, otherwise FileStatus array + */ + public static FileStatus [] listStatus(final FileSystem fs, + final Path dir, final PathFilter filter) throws IOException { + FileStatus [] status = null; + try { + status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter); + } catch (FileNotFoundException fnfe) { + // if directory doesn't exist, return null + if (LOG.isTraceEnabled()) { + LOG.trace(dir + " doesn't exist"); + } + } + if (status == null || status.length < 1) return null; + return status; + } + + /** + * Return the 'path' component of a Path. In Hadoop, Path is an URI. This + * method returns the 'path' component of a Path's URI: e.g. If a Path is + * hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir, + * this method returns /hbase_trunk/TestTable/compaction.dir. + * This method is useful if you want to print out a Path without qualifying + * Filesystem instance. + * @param p Filesystem Path whose 'path' component we are to return. + * @return Path portion of the Filesystem + */ + public static String getPath(Path p) { + return p.toUri().getPath(); + } + + /** + * Given the backup root dir and the backup id, return the log file location for an incremental + * backup. + * @param backupRootDir backup root directory + * @param backupId backup id + * @return logBackupDir: ".../user/biadmin/backup1/WALs/backup_1396650096738" + */ + public static String getLogBackupDir(String backupRootDir, String backupId) { + return backupRootDir + Path.SEPARATOR + backupId+ Path.SEPARATOR + + HConstants.HREGION_LOGDIR_NAME; + } + +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java new file mode 100644 index 0000000..76402c7 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.util; +import java.util.List; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +/** + * Backup set is a named group of HBase tables, + * which are managed together by Backup/Restore + * framework. Instead of using list of tables in backup or restore + * operation, one can use set's name instead. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class BackupSet { + private final String name; + private final List tables; + + public BackupSet(String name, List tables) { + this.name = name; + this.tables = tables; + } + + public String getName() { + return name; + } + + public List getTables() { + return tables; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name).append("={"); + for (int i = 0; i < tables.size(); i++) { + sb.append(tables.get(i)); + if (i < tables.size() - 1) { + sb.append(","); + } + } + sb.append("}"); + return sb.toString(); + } + +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 02ffc18..f46c855 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -914,34 +914,6 @@ public interface Admin extends Abortable, Closeable { void splitRegion(final byte[] regionName, final byte[] splitPoint) throws IOException; - /** - * Backs up given list of tables fully. Asynchronous operation. This means that - * it may be a while before all your tables are backed up. - * You can use Future.get(long, TimeUnit) to wait on the operation to complete. - * - * @param request BackupRequest instance which contains the following members: - * type whether the backup is full or incremental - * tableList list of tables to backup - * targetRootDir root dir for saving the backup - * workers number of paralle workers. -1 - system defined - * bandwidth bandwidth per worker in MB per sec. -1 - unlimited - * @return the result of the async backup. You can use Future.get(long, TimeUnit) to wait on the - * operation to complete - */ - public Future backupTablesAsync(final BackupRequest request) throws IOException; - - /** - * Backs up given list of tables fully. Synchronous operation. - * - * @param request BackupRequest instance which contains the following members: - * type whether the backup is full or incremental - * tableList list of tables to backup - * targetRootDir root dir for saving the backup - * workers number of paralle workers. -1 - system defined - * bandwidth bandwidth per worker in MB per sec. -1 - unlimited - * @return the backup Id - */ - public String backupTables(final BackupRequest request) throws IOException; /** * Modify an existing table, more IRB friendly version. Asynchronous operation. This means that @@ -1727,6 +1699,13 @@ public interface Admin extends Abortable, Closeable { boolean isSplitOrMergeEnabled(final MasterSwitchType switchType) throws IOException; /** + * Get Backup Admin interface + * @return backup admin object + * @throws IOException exception + */ + BackupAdmin getBackupAdmin() throws IOException; + + /** * Currently, there are only two compact types: * {@code NORMAL} means do store files compaction; * {@code MOB} means do mob files compaction. diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java new file mode 100644 index 0000000..7a411cb --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.Future; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupRequest; +import org.apache.hadoop.hbase.backup.RestoreRequest; +import org.apache.hadoop.hbase.backup.util.BackupSet; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +/** + * The administrative API for HBase Backup. Obtain an instance from + * an {@link Admin#getBackupAdmin()} and call {@link #close()} afterwards. + *

BackupAdmin can be used to create backups, restore data from backups and for + * other backup-related operations. + * + * @see Admin + * @since 2.0 + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving + +public interface BackupAdmin extends Closeable{ + + /** + * Backs up given list of tables fully. Synchronous operation. + * + * @param request BackupRequest instance which contains the following members: + * type whether the backup is full or incremental + * tableList list of tables to backup + * targetRootDir root directory for saving the backup + * workers number of parallel workers. -1 - system defined + * bandwidth bandwidth per worker in MB per second. -1 - unlimited + * @return the backup Id + */ + + public String backupTables(final BackupRequest userRequest) throws IOException; + + /** + * Backs up given list of tables fully. Asynchronous operation. + * + * @param request BackupRequest instance which contains the following members: + * type whether the backup is full or incremental + * tableList list of tables to backup + * targetRootDir root dir for saving the backup + * workers number of paralle workers. -1 - system defined + * bandwidth bandwidth per worker in MB per sec. -1 - unlimited + * @return the backup Id future + */ + public Future backupTablesAsync(final BackupRequest userRequest) throws IOException; + + /** + * Describe backup image command + * @param backupId - backup id + * @return backup info + * @throws IOException exception + */ + public BackupInfo getBackupInfo(String backupId) throws IOException; + + /** + * Show backup progress command + * @param backupId - backup id (may be null) + * @return backup progress (0-100%), -1 if no active sessions + * or session not found + * @throws IOException exception + */ + public int getProgress(String backupId) throws IOException; + + /** + * Delete backup image command + * @param backupIds - backup id + * @return total number of deleted sessions + * @throws IOException exception + */ + public int deleteBackups(String[] backupIds) throws IOException; + + /** + * Show backup history command + * @param n - last n backup sessions + * @return list of backup infos + * @throws IOException exception + */ + public List getHistory(int n) throws IOException; + + /** + * Backup sets list command - list all backup sets. Backup set is + * a named group of tables. + * @return all registered backup sets + * @throws IOException exception + */ + public List listBackupSets() throws IOException; + + /** + * Backup set describe command. Shows list of tables in + * this particular backup set. + * @param name set name + * @return backup set description or null + * @throws IOException exception + */ + public BackupSet getBackupSet(String name) throws IOException; + + /** + * Delete backup set command + * @param name - backup set name + * @return true, if success, false - otherwise + * @throws IOException exception + */ + public boolean deleteBackupSet(String name) throws IOException; + + /** + * Add tables to backup set command + * @param name - name of backup set. + * @param tables - list of tables to be added to this set. + * @throws IOException exception + */ + public void addToBackupSet(String name, TableName[] tables) throws IOException; + + /** + * Remove tables from backup set + * @param name - name of backup set. + * @param tables - list of tables to be removed from this set. + * @throws IOException exception + */ + public void removeFromBackupSet(String name, String[] tables) throws IOException; + + /** + * Restore backup + * @param request - restore request + * @throws IOException exception + */ + public void restore(RestoreRequest request) throws IOException; + +} diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 3cc846c..8ba26ba 100644 --- hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -62,7 +62,7 @@ import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupClientUtil; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.security.SecurityCapability; @@ -1565,8 +1565,8 @@ public class HBaseAdmin implements Admin { ProtobufUtil.split(admin, hri, splitPoint); } - @Override - public Future backupTablesAsync(final BackupRequest userRequest) throws IOException { + + Future backupTablesAsync(final BackupRequest userRequest) throws IOException { BackupClientUtil.checkTargetDir(userRequest.getTargetRootDir(), conf); if (userRequest.getTableList() != null) { for (TableName table : userRequest.getTableList()) { @@ -1575,7 +1575,6 @@ public class HBaseAdmin implements Admin { } } } - BackupTablesResponse response = executeCallable( new MasterCallable(getConnection()) { @Override @@ -1585,12 +1584,11 @@ public class HBaseAdmin implements Admin { userRequest.getWorkers(), userRequest.getBandwidth()); return master.backupTables(null, request); } - }); + }, (int) backupWaitTimeout); return new TableBackupFuture(this, TableName.BACKUP_TABLE_NAME, response); } - @Override - public String backupTables(final BackupRequest userRequest) throws IOException { + String backupTables(final BackupRequest userRequest) throws IOException { return get( backupTablesAsync(userRequest), backupWaitTimeout, @@ -2744,6 +2742,13 @@ public class HBaseAdmin implements Admin { return executeCallable(callable, rpcCallerFactory, operationTimeout); } + + private & Closeable, V> V executeCallable(C callable, + int operationTimeout) + throws IOException { + return executeCallable(callable, rpcCallerFactory, operationTimeout); + } + private static & Closeable, V> V executeCallable(C callable, RpcRetryingCallerFactory rpcCallerFactory, int operationTimeout) throws IOException { RpcRetryingCaller caller = rpcCallerFactory.newCaller(); @@ -3491,4 +3496,9 @@ public class HBaseAdmin implements Admin { HConstants.EMPTY_END_ROW, false, 0); } + @Override + public BackupAdmin getBackupAdmin() throws IOException { + return new HBaseBackupAdmin(this); + } + } diff --git hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java new file mode 100644 index 0000000..d2ec108 --- /dev/null +++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; +import org.apache.hadoop.hbase.backup.BackupRequest; +import org.apache.hadoop.hbase.backup.BackupRestoreClientFactory; +import org.apache.hadoop.hbase.backup.RestoreClient; +import org.apache.hadoop.hbase.backup.RestoreRequest; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.backup.util.BackupSet; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * The administrative API implementation for HBase Backup . Obtain an instance from + * an {@link Admin#getBackupAdmin()} and call {@link #close()} afterwards. + *

BackupAdmin can be used to create backups, restore data from backups and for + * other backup-related operations. + * + * @see Admin + * @since 2.0 + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving + +public class HBaseBackupAdmin implements BackupAdmin { + private static final Log LOG = LogFactory.getLog(HBaseBackupAdmin.class); + + private final HBaseAdmin admin; + private final Connection conn; + + HBaseBackupAdmin(HBaseAdmin admin) { + this.admin = admin; + this.conn = admin.getConnection(); + } + + + @Override + public void close() throws IOException { + } + + @Override + public BackupInfo getBackupInfo(String backupId) throws IOException { + BackupInfo backupInfo = null; + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + backupInfo = table.readBackupInfo(backupId); + return backupInfo; + } + } + + @Override + public int getProgress(String backupId) throws IOException { + BackupInfo backupInfo = null; + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + if (backupId == null) { + ArrayList recentSessions = + table.getBackupContexts(BackupState.RUNNING); + if (recentSessions.isEmpty()) { + LOG.warn("No ongoing sessions found."); + return -1; + } + // else show status for ongoing session + // must be one maximum + return recentSessions.get(0).getProgress(); + } else { + + backupInfo = table.readBackupInfo(backupId); + if (backupInfo != null) { + return backupInfo.getProgress(); + } else { + LOG.warn("No information found for backupID=" + backupId); + return -1; + } + } + } + } + + @Override + public int deleteBackups(String[] backupIds) throws IOException { + BackupInfo backupInfo = null; + String backupId = null; + int totalDeleted = 0; + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + for (int i = 0; i < backupIds.length; i++) { + backupId = backupIds[i]; + LOG.info("Deleting backup for backupID=" + backupId + " ..."); + backupInfo = table.readBackupInfo(backupId); + if (backupInfo != null) { + BackupClientUtil.cleanupBackupData(backupInfo, admin.getConfiguration()); + table.deleteBackupInfo(backupInfo.getBackupId()); + LOG.info("Delete backup for backupID=" + backupId + " completed."); + totalDeleted++; + } else { + LOG.warn("Delete backup failed: no information found for backupID=" + backupId); + } + } + } + return totalDeleted; + } + + @Override + public List getHistory(int n) throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + List history = table.getBackupHistory(); + if( history.size() <= n) return history; + List list = new ArrayList(); + for(int i=0; i < n; i++){ + list.add(history.get(i)); + } + return list; + } + } + + @Override + public List listBackupSets() throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + List list = table.listBackupSets(); + List bslist = new ArrayList(); + for (String s : list) { + List tables = table.describeBackupSet(s); + bslist.add( new BackupSet(s, tables)); + } + return bslist; + } + } + + @Override + public BackupSet getBackupSet(String name) throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + List list = table.describeBackupSet(name); + return new BackupSet(name, list); + } + } + + @Override + public boolean deleteBackupSet(String name) throws IOException { + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + if(table.describeBackupSet(name) == null) { + return false; + } + table.deleteBackupSet(name); + return true; + } + } + + @Override + public void addToBackupSet(String name, TableName[] tables) throws IOException { + String[] tableNames = new String[tables.length]; + for(int i = 0; i < tables.length; i++){ + tableNames[i] = tables[i].getNameAsString(); + } + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + table.addToBackupSet(name, tableNames); + LOG.info("Added tables ["+StringUtils.join(tableNames, " ")+"] to '" + name + "' backup set"); + } + } + + @Override + public void removeFromBackupSet(String name, String[] tables) throws IOException { + LOG.info("Removing tables ["+ StringUtils.join(tables, " ")+"] from '" + name + "'"); + try (final BackupSystemTable table = new BackupSystemTable(conn)) { + table.removeFromBackupSet(name, tables); + LOG.info("Removing tables ["+ StringUtils.join(tables, " ")+"] from '" + name + "' completed."); + } + } + + @Override + public void restore(RestoreRequest request) throws IOException { + RestoreClient client = BackupRestoreClientFactory.getRestoreClient(admin.getConfiguration()); + client.restore(request.getBackupRootDir(), + request.getBackupId(), + request.isCheck(), + request.isAutorestore(), + request.getFromTables(), + request.getToTables(), + request.isOverwrite()); + + } + + @Override + public String backupTables(final BackupRequest userRequest) + throws IOException { + return admin.backupTables(userRequest); + } + + + @Override + public Future backupTablesAsync(final BackupRequest userRequest) + throws IOException { + return admin.backupTablesAsync(userRequest); + } + + +} diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/backup/BackupType.java hbase-common/src/main/java/org/apache/hadoop/hbase/backup/BackupType.java index e2e3446..aa225a1 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/backup/BackupType.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/backup/BackupType.java @@ -18,6 +18,11 @@ package org.apache.hadoop.hbase.backup; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Evolving public enum BackupType { FULL, INCREMENTAL } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java deleted file mode 100644 index 7b0b454..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.hbase.backup.util.BackupSet; - -public interface BackupClient extends Configurable{ - - /** - * Describe backup image command - * @param backupId - backup id - * @return backup info - * @throws IOException - */ - public BackupInfo getBackupInfo(String backupId) throws IOException; - - /** - * Show backup progress command - * @param backupId - backup id (may be null) - * @return backup progress (0-100%), -1 if no active sessions - * or session not found - * @throws IOException - */ - public int getProgress(String backupId) throws IOException; - - /** - * Delete backup image command - * @param backupIds - backup id - * @return total number of deleted sessions - * @throws IOException - */ - public int deleteBackups(String[] backupIds) throws IOException; - -// /** -// TODO: Phase 3 -// * Cancel current active backup command -// * @param backupId - backup id -// * @throws IOException -// */ -// public void cancelBackup(String backupId) throws IOException; - - /** - * Show backup history command - * @param n - last n backup sessions - * @throws IOException - */ - public List getHistory(int n) throws IOException; - - /** - * Backup sets list command - list all backup sets. Backup set is - * a named group of tables. - * @throws IOException - */ - public List listBackupSets() throws IOException; - - /** - * Backup set describe command. Shows list of tables in - * this particular backup set. - * @param name set name - * @return backup set description or null - * @throws IOException - */ - public BackupSet getBackupSet(String name) throws IOException; - - /** - * Delete backup set command - * @param name - backup set name - * @return true, if success, false - otherwise - * @throws IOException - */ - public boolean deleteBackupSet(String name) throws IOException; - - /** - * Add tables to backup set command - * @param name - name of backup set. - * @param tables - list of tables to be added to this set. - * @throws IOException - */ - public void addToBackupSet(String name, String[] tablesOrNamespaces) throws IOException; - - /** - * Remove tables from backup set - * @param name - name of backup set. - * @param tables - list of tables to be removed from this set. - * @throws IOException - */ - public void removeFromBackupSet(String name, String[] tablesOrNamepsaces) throws IOException; - - } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java new file mode 100644 index 0000000..26e20f1 --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.backup.impl.BackupManager; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface BackupCopyService extends Configurable { + static enum Type { + FULL, INCREMENTAL + } + + /** + * Copy backup data + * @param backupContext - context + * @param backupManager - manager + * @param conf - configuration + * @param copyType - copy type + * @param options - array of options (implementation-specific) + * @return result (0 - success) + * @throws IOException + */ + public int copy(BackupInfo backupContext, BackupManager backupManager, Configuration conf, + BackupCopyService.Type copyType, String[] options) throws IOException; + + + /** + * Cancel copy job + * @param jobHandler - copy job handler + * @throws IOException + */ + public void cancelCopyJob(String jobHandler) throws IOException; +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java index 182c402..f692bd0 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java @@ -27,12 +27,16 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.backup.impl.BackupCommands; import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants; import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants.BackupCommand; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.LogUtils; import org.apache.hadoop.util.ToolRunner; import org.apache.log4j.Level; import org.apache.log4j.Logger; +@InterfaceAudience.Private +@InterfaceStability.Evolving public class BackupDriver extends AbstractHBaseTool { private static final Log LOG = LogFactory.getLog(BackupDriver.class); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java deleted file mode 100644 index 1ed95f4..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java +++ /dev/null @@ -1,483 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.BackupInfo.Builder; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.TableBackupStatus; - - -/** - * An object to encapsulate the information for each backup request - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class BackupInfo implements Comparable { - private static final Log LOG = LogFactory.getLog(BackupInfo.class); - // backup status flag - public static enum BackupState { - WAITING, RUNNING, COMPLETE, FAILED, CANCELLED; - } - // backup phase - public static enum BackupPhase { - SNAPSHOTCOPY, INCREMENTAL_COPY, STORE_MANIFEST; - } - - // backup id: a timestamp when we request the backup - private String backupId; - - // backup type, full or incremental - private BackupType type; - - // target root directory for storing the backup files - private String targetRootDir; - - // overall backup state - private BackupState state; - - // overall backup phase - private BackupPhase phase; - - // overall backup failure message - private String failedMsg; - - // backup status map for all tables - private Map backupStatusMap; - - // actual start timestamp of the backup process - private long startTs; - - // actual end timestamp of the backup process, could be fail or complete - private long endTs; - - // the total bytes of incremental logs copied - private long totalBytesCopied; - - // for incremental backup, the location of the backed-up hlogs - private String hlogTargetDir = null; - - // incremental backup file list - transient private List incrBackupFileList; - - // new region server log timestamps for table set after distributed log roll - // key - table name, value - map of RegionServer hostname -> last log rolled timestamp - transient private HashMap> tableSetTimestampMap; - - // backup progress in %% (0-100) - private int progress; - - // distributed job id - private String jobId; - - // Number of parallel workers. -1 - system defined - private int workers = -1; - - // Bandwidth per worker in MB per sec. -1 - unlimited - private long bandwidth = -1; - - public BackupInfo() { - } - - public BackupInfo(String backupId, BackupType type, TableName[] tables, String targetRootDir) { - backupStatusMap = new HashMap(); - - this.backupId = backupId; - this.type = type; - this.targetRootDir = targetRootDir; - if(LOG.isDebugEnabled()){ - LOG.debug("CreateBackupContext: " + tables.length+" "+tables[0] ); - } - this.addTables(tables); - - if (type == BackupType.INCREMENTAL) { - setHlogTargetDir(HBackupFileSystem.getLogBackupDir(targetRootDir, backupId)); - } - - this.startTs = 0; - this.endTs = 0; - } - - public String getJobId() { - return jobId; - } - - public void setJobId(String jobId) { - this.jobId = jobId; - } - - public int getWorkers() { - return workers; - } - - public void setWorkers(int workers) { - this.workers = workers; - } - - public long getBandwidth() { - return bandwidth; - } - - public void setBandwidth(long bandwidth) { - this.bandwidth = bandwidth; - } - - public void setBackupStatusMap(Map backupStatusMap) { - this.backupStatusMap = backupStatusMap; - } - - public HashMap> getTableSetTimestampMap() { - return tableSetTimestampMap; - } - - public void setTableSetTimestampMap(HashMap> tableSetTimestampMap) { - this.tableSetTimestampMap = tableSetTimestampMap; - } - - public String getHlogTargetDir() { - return hlogTargetDir; - } - - public void setType(BackupType type) { - this.type = type; - } - - public void setTargetRootDir(String targetRootDir) { - this.targetRootDir = targetRootDir; - } - - public void setTotalBytesCopied(long totalBytesCopied) { - this.totalBytesCopied = totalBytesCopied; - } - - public void setCancelled(boolean cancelled) { - this.state = BackupState.CANCELLED;; - } - - /** - * Set progress (0-100%) - * @param msg progress value - */ - - public void setProgress(int p) { - this.progress = p; - } - - /** - * Get current progress - */ - public int getProgress() { - return progress; - } - - - /** - * Has been marked as cancelled or not. - * @return True if marked as cancelled - */ - public boolean isCancelled() { - return this.state == BackupState.CANCELLED; - } - - public String getBackupId() { - return backupId; - } - - public void setBackupId(String backupId) { - this.backupId = backupId; - } - - public BackupStatus getBackupStatus(TableName table) { - return this.backupStatusMap.get(table); - } - - public String getFailedMsg() { - return failedMsg; - } - - public void setFailedMsg(String failedMsg) { - this.failedMsg = failedMsg; - } - - public long getStartTs() { - return startTs; - } - - public void setStartTs(long startTs) { - this.startTs = startTs; - } - - public long getEndTs() { - return endTs; - } - - public void setEndTs(long endTs) { - this.endTs = endTs; - } - - public long getTotalBytesCopied() { - return totalBytesCopied; - } - - public BackupState getState() { - return state; - } - - public void setState(BackupState flag) { - this.state = flag; - } - - public BackupPhase getPhase() { - return phase; - } - - public void setPhase(BackupPhase phase) { - this.phase = phase; - } - - public BackupType getType() { - return type; - } - - public void setSnapshotName(TableName table, String snapshotName) { - this.backupStatusMap.get(table).setSnapshotName(snapshotName); - } - - public String getSnapshotName(TableName table) { - return this.backupStatusMap.get(table).getSnapshotName(); - } - - public List getSnapshotNames() { - List snapshotNames = new ArrayList(); - for (BackupStatus backupStatus : this.backupStatusMap.values()) { - snapshotNames.add(backupStatus.getSnapshotName()); - } - return snapshotNames; - } - - public Set getTables() { - return this.backupStatusMap.keySet(); - } - - public List getTableNames() { - return new ArrayList(backupStatusMap.keySet()); - } - - public void addTables(TableName[] tables) { - for (TableName table : tables) { - BackupStatus backupStatus = new BackupStatus(table, this.targetRootDir, this.backupId); - this.backupStatusMap.put(table, backupStatus); - } - } - - public String getTargetRootDir() { - return targetRootDir; - } - - public void setHlogTargetDir(String hlogTagetDir) { - this.hlogTargetDir = hlogTagetDir; - } - - public String getHLogTargetDir() { - return hlogTargetDir; - } - - public List getIncrBackupFileList() { - return incrBackupFileList; - } - - public void setIncrBackupFileList(List incrBackupFileList) { - this.incrBackupFileList = incrBackupFileList; - } - - /** - * Set the new region server log timestamps after distributed log roll - * @param newTableSetTimestampMap table timestamp map - */ - public void setIncrTimestampMap(HashMap> newTableSetTimestampMap) { - this.tableSetTimestampMap = newTableSetTimestampMap; - } - - /** - * Get new region server log timestamps after distributed log roll - * @return new region server log timestamps - */ - public HashMap> getIncrTimestampMap() { - return this.tableSetTimestampMap; - } - - public TableName getTableBySnapshot(String snapshotName) { - for (Entry entry : this.backupStatusMap.entrySet()) { - if (snapshotName.equals(entry.getValue().getSnapshotName())) { - return entry.getKey(); - } - } - return null; - } - - public BackupProtos.BackupInfo toProtosBackupInfo() { - BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder(); - builder.setBackupId(getBackupId()); - setBackupStatusMap(builder); - builder.setEndTs(getEndTs()); - if (getFailedMsg() != null) { - builder.setFailedMessage(getFailedMsg()); - } - if (getState() != null) { - builder.setState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name())); - } - if (getPhase() != null) { - builder.setPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name())); - } - - builder.setProgress(getProgress()); - builder.setStartTs(getStartTs()); - builder.setTargetRootDir(getTargetRootDir()); - builder.setType(BackupProtos.BackupType.valueOf(getType().name())); - builder.setWorkersNumber(workers); - builder.setBandwidth(bandwidth); - if (jobId != null) { - builder.setJobId(jobId); - } - return builder.build(); - } - - public byte[] toByteArray() throws IOException { - return toProtosBackupInfo().toByteArray(); - } - - private void setBackupStatusMap(Builder builder) { - for (Entry entry: backupStatusMap.entrySet()) { - builder.addTableBackupStatus(entry.getValue().toProto()); - } - } - - public static BackupInfo fromByteArray(byte[] data) throws IOException { - return fromProto(BackupProtos.BackupInfo.parseFrom(data)); - } - - public static BackupInfo fromStream(final InputStream stream) throws IOException { - return fromProto(BackupProtos.BackupInfo.parseDelimitedFrom(stream)); - } - - public static BackupInfo fromProto(BackupProtos.BackupInfo proto) { - BackupInfo context = new BackupInfo(); - context.setBackupId(proto.getBackupId()); - context.setBackupStatusMap(toMap(proto.getTableBackupStatusList())); - context.setEndTs(proto.getEndTs()); - if (proto.hasFailedMessage()) { - context.setFailedMsg(proto.getFailedMessage()); - } - if (proto.hasState()) { - context.setState(BackupInfo.BackupState.valueOf(proto.getState().name())); - } - - context.setHlogTargetDir(HBackupFileSystem.getLogBackupDir(proto.getTargetRootDir(), - proto.getBackupId())); - - if (proto.hasPhase()) { - context.setPhase(BackupPhase.valueOf(proto.getPhase().name())); - } - if (proto.hasProgress()) { - context.setProgress(proto.getProgress()); - } - context.setStartTs(proto.getStartTs()); - context.setTargetRootDir(proto.getTargetRootDir()); - context.setType(BackupType.valueOf(proto.getType().name())); - context.setWorkers(proto.getWorkersNumber()); - context.setBandwidth(proto.getBandwidth()); - if (proto.hasJobId()) { - context.setJobId(proto.getJobId()); - } - return context; - } - - private static Map toMap(List list) { - HashMap map = new HashMap<>(); - for (TableBackupStatus tbs : list){ - map.put(ProtobufUtil.toTableName(tbs.getTable()), BackupStatus.convert(tbs)); - } - return map; - } - - public String getShortDescription() { - StringBuilder sb = new StringBuilder(); - sb.append("ID : " + backupId).append("\n"); - sb.append("Tables : " + getTableListAsString()).append("\n"); - sb.append("State : " + getState()).append("\n"); - Date date = null; - Calendar cal = Calendar.getInstance(); - cal.setTimeInMillis(getStartTs()); - date = cal.getTime(); - sb.append("Start time : " + date).append("\n"); - if (state == BackupState.FAILED) { - sb.append("Failed message : " + getFailedMsg()).append("\n"); - } else if (state == BackupState.RUNNING) { - sb.append("Phase : " + getPhase()).append("\n"); - } else if (state == BackupState.COMPLETE) { - cal = Calendar.getInstance(); - cal.setTimeInMillis(getEndTs()); - date = cal.getTime(); - sb.append("End time : " + date).append("\n"); - } - sb.append("Progress : " + getProgress()).append("\n"); - return sb.toString(); - } - - public String getStatusAndProgressAsString() { - StringBuilder sb = new StringBuilder(); - sb.append("id: ").append(getBackupId()).append(" state: ").append(getState()) - .append(" progress: ").append(getProgress()); - return sb.toString(); - } - - public String getTableListAsString() { - return StringUtils.join(backupStatusMap.keySet(), ";"); - } - - @Override - public int compareTo(BackupInfo o) { - Long thisTS = - new Long(this.getBackupId().substring(this.getBackupId().lastIndexOf("_") + 1)); - Long otherTS = - new Long(o.getBackupId().substring(o.getBackupId().lastIndexOf("_") + 1)); - return thisTS.compareTo(otherTS); - } - - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreFactory.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreFactory.java deleted file mode 100644 index 30882bd..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreFactory.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.backup.impl.BackupClientImpl; -import org.apache.hadoop.hbase.backup.impl.BackupCopyService; -import org.apache.hadoop.hbase.backup.impl.IncrementalRestoreService; -import org.apache.hadoop.hbase.backup.impl.RestoreClientImpl; -import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyService; -import org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreService; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.util.ReflectionUtils; - -@InterfaceAudience.Private -@InterfaceStability.Evolving -public final class BackupRestoreFactory { - - public final static String HBASE_INCR_RESTORE_IMPL_CLASS = "hbase.incremental.restore.class"; - public final static String HBASE_BACKUP_COPY_IMPL_CLASS = "hbase.backup.copy.class"; - public final static String HBASE_BACKUP_CLIENT_IMPL_CLASS = "hbase.backup.client.class"; - public final static String HBASE_RESTORE_CLIENT_IMPL_CLASS = "hbase.restore.client.class"; - - private BackupRestoreFactory(){ - throw new AssertionError("Instantiating utility class..."); - } - - /** - * Gets incremental restore service - * @param conf - configuration - * @return incremental backup service instance - */ - public static IncrementalRestoreService getIncrementalRestoreService(Configuration conf) { - Class cls = - conf.getClass(HBASE_INCR_RESTORE_IMPL_CLASS, MapReduceRestoreService.class, - IncrementalRestoreService.class); - IncrementalRestoreService service = ReflectionUtils.newInstance(cls, conf); - service.setConf(conf); - return service; - } - - /** - * Gets backup copy service - * @param conf - configuration - * @return backup copy service - */ - public static BackupCopyService getBackupCopyService(Configuration conf) { - Class cls = - conf.getClass(HBASE_BACKUP_COPY_IMPL_CLASS, MapReduceBackupCopyService.class, - BackupCopyService.class); - BackupCopyService service = ReflectionUtils.newInstance(cls, conf);; - service.setConf(conf); - return service; - } - /** - * Gets backup client implementation - * @param conf - configuration - * @return backup client - */ - public static BackupClient getBackupClient(Configuration conf) { - Class cls = - conf.getClass(HBASE_BACKUP_CLIENT_IMPL_CLASS, BackupClientImpl.class, - BackupClient.class); - BackupClient client = ReflectionUtils.newInstance(cls, conf); - client.setConf(conf); - return client; - } - - /** - * Gets restore client implementation - * @param conf - configuration - * @return backup client - */ - public static RestoreClient getRestoreClient(Configuration conf) { - Class cls = - conf.getClass(HBASE_RESTORE_CLIENT_IMPL_CLASS, RestoreClientImpl.class, - RestoreClient.class); - RestoreClient client = ReflectionUtils.newInstance(cls, conf); - client.setConf(conf); - return client; - } -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServerFactory.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServerFactory.java new file mode 100644 index 0000000..25ec9d9 --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServerFactory.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyService; +import org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreService; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.util.ReflectionUtils; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class BackupRestoreServerFactory { + + public final static String HBASE_INCR_RESTORE_IMPL_CLASS = "hbase.incremental.restore.class"; + public final static String HBASE_BACKUP_COPY_IMPL_CLASS = "hbase.backup.copy.class"; + + private BackupRestoreServerFactory(){ + throw new AssertionError("Instantiating utility class..."); + } + + /** + * Gets incremental restore service + * @param conf - configuration + * @return incremental backup service instance + */ + public static IncrementalRestoreService getIncrementalRestoreService(Configuration conf) { + Class cls = + conf.getClass(HBASE_INCR_RESTORE_IMPL_CLASS, MapReduceRestoreService.class, + IncrementalRestoreService.class); + IncrementalRestoreService service = ReflectionUtils.newInstance(cls, conf); + service.setConf(conf); + return service; + } + + /** + * Gets backup copy service + * @param conf - configuration + * @return backup copy service + */ + public static BackupCopyService getBackupCopyService(Configuration conf) { + Class cls = + conf.getClass(HBASE_BACKUP_COPY_IMPL_CLASS, MapReduceBackupCopyService.class, + BackupCopyService.class); + BackupCopyService service = ReflectionUtils.newInstance(cls, conf);; + service.setConf(conf); + return service; + } +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java deleted file mode 100644 index 1564b04..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.Serializable; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; - -/** - * Backup status and related information encapsulated for a table. - * At this moment only TargetDir and SnapshotName is encapsulated here. - */ - -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class BackupStatus implements Serializable { - - private static final long serialVersionUID = -5968397963548535982L; - - // table name for backup - private TableName table; - - // target directory of the backup image for this table - private String targetDir; - - // snapshot name for offline/online snapshot - private String snapshotName = null; - - public BackupStatus() { - - } - - public BackupStatus(TableName table, String targetRootDir, String backupId) { - this.table = table; - this.targetDir = HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table); - } - - public String getSnapshotName() { - return snapshotName; - } - - public void setSnapshotName(String snapshotName) { - this.snapshotName = snapshotName; - } - - public String getTargetDir() { - return targetDir; - } - - public TableName getTable() { - return table; - } - - public void setTable(TableName table) { - this.table = table; - } - - public void setTargetDir(String targetDir) { - this.targetDir = targetDir; - } - - public static BackupStatus convert(BackupProtos.TableBackupStatus proto) - { - BackupStatus bs = new BackupStatus(); - bs.setTable(ProtobufUtil.toTableName(proto.getTable())); - bs.setTargetDir(proto.getTargetDir()); - if(proto.hasSnapshot()){ - bs.setSnapshotName(proto.getSnapshot()); - } - return bs; - } - - public BackupProtos.TableBackupStatus toProto() { - BackupProtos.TableBackupStatus.Builder builder = - BackupProtos.TableBackupStatus.newBuilder(); - if(snapshotName != null) { - builder.setSnapshot(snapshotName); - } - builder.setTable(ProtobufUtil.toProtoTableName(table)); - builder.setTargetDir(targetDir); - return builder.build(); - } -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java new file mode 100644 index 0000000..ae48480 --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface IncrementalRestoreService extends Configurable{ + + /** + * Run restore operation + * @param logDirectoryPaths - path array of WAL log directories + * @param fromTables - from tables + * @param toTables - to tables + * @throws IOException + */ + public void run(Path[] logDirectoryPaths, TableName[] fromTables, TableName[] toTables) + throws IOException; +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java deleted file mode 100644 index a3aaa98..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; - -public interface RestoreClient { - - public void setConf(Configuration conf); - - /** - * Restore operation. - * @param backupRootDir The root dir for backup image - * @param backupId The backup id for image to be restored - * @param check True if only do dependency check - * @param autoRestore True if automatically restore following the dependency - * @param sTableArray The array of tables to be restored - * @param tTableArray The array of mapping tables to restore to - * @param isOverwrite True then do restore overwrite if target table exists, otherwise fail the - * request if target table exists - * @return True if only do dependency check - * @throws IOException if any failure during restore - */ - public boolean restore( - String backupRootDir, - String backupId, boolean check, boolean autoRestore, TableName[] sTableArray, - TableName[] tTableArray, boolean isOverwrite) throws IOException; -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java index 6739b5c..83c8297 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java @@ -25,7 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupUtil; +import org.apache.hadoop.hbase.backup.util.BackupServerUtil; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.LogUtils; import org.apache.hadoop.util.ToolRunner; @@ -112,19 +112,19 @@ public class RestoreDriver extends AbstractHBaseTool { // parse main restore command options String[] remainArgs = cmd.getArgs(); - if (remainArgs.length < 4) { + if (remainArgs.length < 3) { System.out.println(USAGE); return -1; } - String backupRootDir = remainArgs[1]; - String backupId = remainArgs[2]; - String tables = remainArgs[3]; + String backupRootDir = remainArgs[0]; + String backupId = remainArgs[1]; + String tables = remainArgs[2]; - String tableMapping = (remainArgs.length > 4) ? remainArgs[4] : null; + String tableMapping = (remainArgs.length > 3) ? remainArgs[3] : null; - TableName[] sTableArray = BackupUtil.parseTableNames(tables); - TableName[] tTableArray = BackupUtil.parseTableNames(tableMapping); + TableName[] sTableArray = BackupServerUtil.parseTableNames(tables); + TableName[] tTableArray = BackupServerUtil.parseTableNames(tableMapping); if (sTableArray != null && tTableArray != null && (sTableArray.length != tTableArray.length)) { System.out.println("ERROR: table mapping mismatch: " + tables + " : " + tableMapping); @@ -133,7 +133,7 @@ public class RestoreDriver extends AbstractHBaseTool { } - RestoreClient client = BackupRestoreFactory.getRestoreClient(getConf()); + RestoreClient client = BackupRestoreClientFactory.getRestoreClient(getConf()); try{ client.restore(backupRootDir, backupId, check, autoRestore, sTableArray, tTableArray, isOverwrite); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupClientImpl.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupClientImpl.java deleted file mode 100644 index 7b2db3d..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupClientImpl.java +++ /dev/null @@ -1,231 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupClient; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.util.BackupSet; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; - -/** - * Backup HBase tables locally or on a remote cluster Serve as client entry point for the following - * features: - Full Backup provide local and remote back/restore for a list of tables - Incremental - * backup to build on top of full backup as daily/weekly backup - Convert incremental backup WAL - * files into hfiles - Merge several backup images into one(like merge weekly into monthly) - Add - * and remove table to and from Backup image - Cancel a backup process - Full backup based on - * existing snapshot - Describe information of a backup image - */ - -@InterfaceAudience.Public -@InterfaceStability.Evolving -public final class BackupClientImpl implements BackupClient{ - private static final Log LOG = LogFactory.getLog(BackupClientImpl.class); - private Configuration conf; - - public BackupClientImpl() { - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - } - - - @Override - public BackupInfo getBackupInfo(String backupId) throws IOException { - BackupInfo backupInfo = null; - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - backupInfo = table.readBackupInfo(backupId); - return backupInfo; - } - } - - @Override - public int getProgress(String backupId) throws IOException { - BackupInfo backupInfo = null; - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - if (backupId == null) { - ArrayList recentSessions = - table.getBackupContexts(BackupState.RUNNING); - if (recentSessions.isEmpty()) { - LOG.warn("No ongonig sessions found."); - return -1; - } - // else show status for all ongoing sessions - // must be one maximum - return recentSessions.get(0).getProgress(); - } else { - - backupInfo = table.readBackupInfo(backupId); - if (backupInfo != null) { - return backupInfo.getProgress(); - } else { - LOG.warn("No information found for backupID=" + backupId); - return -1; - } - } - } - } - - @Override - public int deleteBackups(String[] backupIds) throws IOException { - BackupInfo backupInfo = null; - String backupId = null; - int totalDeleted = 0; - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - for (int i = 0; i < backupIds.length; i++) { - backupId = backupIds[i]; - backupInfo = table.readBackupInfo(backupId); - if (backupInfo != null) { - BackupUtil.cleanupBackupData(backupInfo, conf); - table.deleteBackupInfo(backupInfo.getBackupId()); - System.out.println("Delete backup for backupID=" + backupId + " completed."); - totalDeleted++; - } else { - System.out.println("Delete backup failed: no information found for backupID=" + backupId); - } - } - } - return totalDeleted; - } - -//TODO: Cancel backup? - -// @Override -// public void cancelBackup(String backupId) throws IOException { -// // Kill distributed job if active -// // Backup MUST not be in COMPLETE state -// try (final BackupSystemTable table = new BackupSystemTable(conf)) { -// BackupContext backupContext = table.readBackupStatus(backupId); -// String errMessage = null; -// if (backupContext != null && backupContext.getState() != BackupState.COMPLETE) { -// BackupUtil.cleanupBackupData(backupContext, conf); -// table.deleteBackupStatus(backupContext.getBackupId()); -// byte[] jobId = backupContext.getJobId(); -// if(jobId != null) { -// BackupCopyService service = BackupRestoreFactory.getBackupCopyService(conf); -// service.cancelCopyJob(jobId); -// } else{ -// errMessage = "Distributed Job ID is null for backup "+backupId + -// " in "+ backupContext.getState() + " state."; -// } -// } else if( backupContext == null){ -// errMessage = "No information found for backupID=" + backupId; -// } else { -// errMessage = "Can not cancel "+ backupId + " in " + backupContext.getState()+" state"; -// } -// -// if( errMessage != null) { -// throw new IOException(errMessage); -// } -// } -// // then clean backup image -// deleteBackups(new String[] { backupId }); -// } - - @Override - public List getHistory(int n) throws IOException { - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - List history = table.getBackupHistory(); - if( history.size() <= n) return history; - List list = new ArrayList(); - for(int i=0; i < n; i++){ - list.add(history.get(i)); - } - return list; - } - } - - - @Override - public List listBackupSets() throws IOException{ - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - List list = table.listBackupSets(); - List bslist = new ArrayList(); - for (String s : list) { - List tables = table.describeBackupSet(s); - bslist.add( new BackupSet(s, tables)); - } - return bslist; - } - } - - - @Override - public BackupSet getBackupSet(String name) throws IOException{ - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - List list = table.describeBackupSet(name); - return new BackupSet(name, list); - } - } - - @Override - public boolean deleteBackupSet(String name) throws IOException { - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - if(table.describeBackupSet(name) == null) { - return false; - } - table.deleteBackupSet(name); - return true; - } - } - - @Override - public void addToBackupSet(String name, String[] tablesOrNamespaces) throws IOException { - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - table.addToBackupSet(name, tablesOrNamespaces); - System.out.println("Added tables to '" + name + "'"); - } - } - - @Override - public void removeFromBackupSet(String name, String[] tablesOrNamepsaces) throws IOException { - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - table.removeFromBackupSet(name, tablesOrNamepsaces); - System.out.println("Removed tables from '" + name + "'"); - } - } - - @Override - public Configuration getConf() { - return conf; - } - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java deleted file mode 100644 index bd03605..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ /dev/null @@ -1,561 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupClient; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupRestoreFactory; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants.BackupCommand; -import org.apache.hadoop.hbase.backup.util.BackupSet; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; - -import com.google.common.collect.Lists; - -/** - * General backup commands, options and usage messages - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public final class BackupCommands { - - private static final String USAGE = "Usage: hbase backup COMMAND\n" - + "where COMMAND is one of:\n" - + " create create a new backup image\n" - + " cancel cancel an ongoing backup\n" - + " delete delete an existing backup image\n" - + " describe show the detailed information of a backup image\n" - + " history show history of all successful backups\n" - + " progress show the progress of the latest backup request\n" - + " set backup set management\n" - + "Enter \'help COMMAND\' to see help message for each command\n"; - - private static final String CREATE_CMD_USAGE = - "Usage: hbase backup create [tables] [-s name] [-convert] " - + "[-silent] [-w workers][-b bandwith]\n" + " type \"full\" to create a full backup image;\n" - + " \"incremental\" to create an incremental backup image\n" - + " backup_root_path The full root path to store the backup image,\n" - + " the prefix can be hdfs, webhdfs or gpfs\n" + " Options:\n" - + " tables If no tables (\"\") are specified, all tables are backed up. " - + "Otherwise it is a\n" + " comma separated list of tables.\n" - + " -s name Use the specified snapshot for full backup\n" - + " -convert For an incremental backup, convert WAL files to HFiles\n" - + " -w number of parallel workers.\n" - + " -b bandwith per one worker (in MB sec)" ; - - private static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress \n" - + " backupId backup image id;\n"; - - private static final String DESCRIBE_CMD_USAGE = "Usage: hbase backup decsribe \n" - + " backupId backup image id\n"; - - private static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [-n N]\n" - + " -n N show up to N last backup sessions, default - 10;\n"; - - private static final String DELETE_CMD_USAGE = "Usage: hbase backup delete \n" - + " backupId backup image id;\n"; - - private static final String CANCEL_CMD_USAGE = "Usage: hbase backup progress \n" - + " backupId backup image id;\n"; - - private static final String SET_CMD_USAGE = "Usage: hbase set COMMAND [name] [tables]\n" - + " name Backup set name\n" - + " tables If no tables (\"\") are specified, all tables will belong to the set. " - + "Otherwise it is a\n" + " comma separated list of tables.\n" - + "where COMMAND is one of:\n" - + " add add tables to a set, crete set if needed\n" - + " remove remove tables from set\n" - + " list list all sets\n" - + " describe describes set\n" - + " delete delete backup set\n"; - - public static abstract class Command extends Configured { - Command(Configuration conf) { - super(conf); - } - public abstract void execute() throws IOException; - } - - private BackupCommands() { - throw new AssertionError("Instantiating utility class..."); - } - - public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) { - Command cmd = null; - switch (type) { - case CREATE: - cmd = new CreateCommand(conf, cmdline); - break; - case DESCRIBE: - cmd = new DescribeCommand(conf, cmdline); - break; - case PROGRESS: - cmd = new ProgressCommand(conf, cmdline); - break; - case DELETE: - cmd = new DeleteCommand(conf, cmdline); - break; - case CANCEL: - cmd = new CancelCommand(conf, cmdline); - break; - case HISTORY: - cmd = new HistoryCommand(conf, cmdline); - break; - case SET: - cmd = new BackupSetCommand(conf, cmdline); - break; - case HELP: - default: - cmd = new HelpCommand(conf, cmdline); - break; - } - return cmd; - } - - - public static class CreateCommand extends Command { - CommandLine cmdline; - - CreateCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null) { - System.out.println("ERROR: missing arguments"); - System.out.println(CREATE_CMD_USAGE); - System.exit(-1); - } - String[] args = cmdline.getArgs(); - if (args.length < 3 || args.length > 4) { - System.out.println("ERROR: wrong number of arguments"); - System.out.println(CREATE_CMD_USAGE); - System.exit(-1); - } - - if (!BackupType.FULL.toString().equalsIgnoreCase(args[1]) - && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) { - System.out.println("ERROR: invalid backup type"); - System.out.println(CREATE_CMD_USAGE); - System.exit(-1); - } - - String tables = null; - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - - // Check backup set - if (cmdline.hasOption("set")) { - String setName = cmdline.getOptionValue("set"); - tables = getTablesForSet(setName, conf); - - if (tables == null) throw new IOException("Backup set '" + setName - + "' is either empty or does not exist"); - } else { - tables = (args.length == 4) ? args[3] : null; - } - int bandwidth = cmdline.hasOption('b') ? Integer.parseInt(cmdline.getOptionValue('b')) : -1; - int workers = cmdline.hasOption('w') ? Integer.parseInt(cmdline.getOptionValue('w')) : -1; - - try (Connection conn = ConnectionFactory.createConnection(getConf()); - Admin admin = conn.getAdmin();) { - BackupRequest request = new BackupRequest(); - request.setBackupType(BackupType.valueOf(args[1].toUpperCase())) - .setTableList(tables != null?Lists.newArrayList(BackupUtil.parseTableNames(tables)): null) - .setTargetRootDir(args[2]).setWorkers(workers).setBandwidth(bandwidth); - admin.backupTables(request); - } catch (IOException e) { - throw e; - } - } - private String getTablesForSet(String name, Configuration conf) - throws IOException { - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - List tables = table.describeBackupSet(name); - if (tables == null) return null; - return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); - } - } - } - - private static class HelpCommand extends Command { - CommandLine cmdline; - - HelpCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null) { - System.out.println(USAGE); - System.exit(0); - } - - String[] args = cmdline.getArgs(); - if (args == null || args.length == 0) { - System.out.println(USAGE); - System.exit(0); - } - - if (args.length != 1) { - System.out.println("Only support check help message of a single command type"); - System.out.println(USAGE); - System.exit(0); - } - - String type = args[0]; - - if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) { - System.out.println(CREATE_CMD_USAGE); - } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) { - System.out.println(DESCRIBE_CMD_USAGE); - } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) { - System.out.println(HISTORY_CMD_USAGE); - } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) { - System.out.println(PROGRESS_CMD_USAGE); - } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) { - System.out.println(DELETE_CMD_USAGE); - } - if (BackupCommand.CANCEL.name().equalsIgnoreCase(type)) { - System.out.println(CANCEL_CMD_USAGE); - } - if (BackupCommand.SET.name().equalsIgnoreCase(type)) { - System.out.println(SET_CMD_USAGE); - } else { - System.out.println("Unknown command : " + type); - System.out.println(USAGE); - } - System.exit(0); - } - } - - private static class DescribeCommand extends Command { - CommandLine cmdline; - - DescribeCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null) { - System.out.println("ERROR: missing arguments"); - System.out.println(DESCRIBE_CMD_USAGE); - System.exit(-1); - } - String[] args = cmdline.getArgs(); - if (args.length != 2) { - System.out.println("ERROR: wrong number of arguments"); - System.out.println(DESCRIBE_CMD_USAGE); - System.exit(-1); - } - - String backupId = args[1]; - try { - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); - BackupInfo info = client.getBackupInfo(backupId); - System.out.println(info.getShortDescription()); - } catch (RuntimeException e) { - System.out.println("ERROR: " + e.getMessage()); - System.exit(-1); - } - } - } - - private static class ProgressCommand extends Command { - CommandLine cmdline; - - ProgressCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null || - cmdline.getArgs().length != 2) { - System.out.println("No backup id was specified, " - + "will retrieve the most recent (ongoing) sessions"); - } - String[] args = cmdline.getArgs(); - if (args.length > 2) { - System.out.println("ERROR: wrong number of arguments: " + args.length); - System.out.println(PROGRESS_CMD_USAGE); - System.exit(-1); - } - - String backupId = args == null ? null : args[1]; - try { - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); - int progress = client.getProgress(backupId); - if(progress < 0){ - System.out.println("No info was found for backup id: "+backupId); - } else{ - System.out.println(backupId+" progress=" + progress+"%"); - } - } catch (RuntimeException e) { - System.out.println("ERROR: " + e.getMessage()); - System.exit(-1); - } - } - } - - private static class DeleteCommand extends Command { - - CommandLine cmdline; - DeleteCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null || - cmdline.getArgs().length < 2) { - System.out.println("No backup id(s) was specified"); - System.out.println(PROGRESS_CMD_USAGE); - System.exit(-1); - } - String[] args = cmdline.getArgs(); - - String[] backupIds = new String[args.length-1]; - System.arraycopy(args, 1, backupIds, 0, backupIds.length); - try { - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); - client.deleteBackups(args); - } catch (RuntimeException e) { - System.out.println("ERROR: " + e.getMessage()); - System.exit(-1); - } - } - } - -// TODO Cancel command - - private static class CancelCommand extends Command { - CommandLine cmdline; - - CancelCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - if (cmdline == null || - cmdline.getArgs() == null || cmdline.getArgs().length < 2) { - System.out.println("No backup id(s) was specified, will use the most recent one"); - } - String[] args = cmdline.getArgs(); - String backupId = args == null || args.length == 0 ? null : args[1]; - try { - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); -//TODO -// client.cancelBackup(backupId); - } catch (RuntimeException e) { - System.out.println("ERROR: " + e.getMessage()); - System.exit(-1); - } - } - } - - private static class HistoryCommand extends Command { - CommandLine cmdline; - private final static int DEFAULT_HISTORY_LENGTH = 10; - - HistoryCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - - int n = parseHistoryLength(); - try { - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); - List history = client.getHistory(n); - for(BackupInfo info: history){ - System.out.println(info.getShortDescription()); - } - } catch (RuntimeException e) { - System.out.println("ERROR: " + e.getMessage()); - System.exit(-1); - } - } - - private int parseHistoryLength() { - String value = cmdline.getOptionValue("n"); - if (value == null) return DEFAULT_HISTORY_LENGTH; - return Integer.parseInt(value); - } - } - - private static class BackupSetCommand extends Command { - private final static String SET_ADD_CMD = "add"; - private final static String SET_REMOVE_CMD = "remove"; - private final static String SET_DELETE_CMD = "delete"; - private final static String SET_DESCRIBE_CMD = "describe"; - private final static String SET_LIST_CMD = "list"; - - CommandLine cmdline; - - BackupSetCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - - // Command-line must have at least one element - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { - throw new IOException("command line format"); - } - String[] args = cmdline.getArgs(); - String cmdStr = args[1]; - BackupCommand cmd = getCommand(cmdStr); - - try { - - switch (cmd) { - case SET_ADD: - processSetAdd(args); - break; - case SET_REMOVE: - processSetRemove(args); - break; - case SET_DELETE: - processSetDelete(args); - break; - case SET_DESCRIBE: - processSetDescribe(args); - break; - case SET_LIST: - processSetList(args); - break; - default: - break; - - } - } catch (RuntimeException e) { - System.out.println("ERROR: " + e.getMessage()); - System.exit(-1); - } - } - - private void processSetList(String[] args) throws IOException { - // List all backup set names - // does not expect any args - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); - client.listBackupSets(); - } - - private void processSetDescribe(String[] args) throws IOException { - if (args == null || args.length != 3) { - throw new RuntimeException("Wrong number of args: "+args.length); - } - String setName = args[2]; - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); - BackupSet set = client.getBackupSet(setName); - System.out.println(set); - } - - private void processSetDelete(String[] args) throws IOException { - if (args == null || args.length != 3) { - throw new RuntimeException("Wrong number of args"); - } - String setName = args[2]; - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); - boolean result = client.deleteBackupSet(setName); - if(result){ - System.out.println("Delete set "+setName+" OK."); - } else{ - System.out.println("Set "+setName+" does not exists"); - } - } - - private void processSetRemove(String[] args) throws IOException { - if (args == null || args.length != 4) { - throw new RuntimeException("Wrong args"); - } - String setName = args[2]; - String[] tables = args[3].split(","); - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); - client.removeFromBackupSet(setName, tables); - } - - private void processSetAdd(String[] args) throws IOException { - if (args == null || args.length != 4) { - throw new RuntimeException("Wrong args"); - } - String setName = args[2]; - String[] tables = args[3].split(","); - Configuration conf = getConf() != null? getConf():HBaseConfiguration.create(); - BackupClient client = BackupRestoreFactory.getBackupClient(conf); - client.addToBackupSet(setName, tables); - } - - private BackupCommand getCommand(String cmdStr) throws IOException { - if (cmdStr.equals(SET_ADD_CMD)) { - return BackupCommand.SET_ADD; - } else if (cmdStr.equals(SET_REMOVE_CMD)) { - return BackupCommand.SET_REMOVE; - } else if (cmdStr.equals(SET_DELETE_CMD)) { - return BackupCommand.SET_DELETE; - } else if (cmdStr.equals(SET_DESCRIBE_CMD)) { - return BackupCommand.SET_DESCRIBE; - } else if (cmdStr.equals(SET_LIST_CMD)) { - return BackupCommand.SET_LIST; - } else { - throw new IOException("Unknown command for 'set' :" + cmdStr); - } - } - - } - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCopyService.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCopyService.java deleted file mode 100644 index a738b5d..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCopyService.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -@InterfaceAudience.Private -@InterfaceStability.Evolving -public interface BackupCopyService extends Configurable { - static enum Type { - FULL, INCREMENTAL - } - - /** - * Copy backup data - * @param backupContext - context - * @param backupManager - manager - * @param conf - configuration - * @param copyType - copy type - * @param options - list of options - * @return result (0 - success) - * @throws IOException - */ - public int copy(BackupInfo backupContext, BackupManager backupManager, Configuration conf, - BackupCopyService.Type copyType, String[] options) throws IOException; - - - /** - * Cancel copy job - * @param jobHandler - copy job handler - * @throws IOException - */ - public void cancelCopyJob(String jobHandler) throws IOException; -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java deleted file mode 100644 index ca204b4..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import org.apache.hadoop.hbase.HBaseIOException; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -/** - * Backup exception - */ -@SuppressWarnings("serial") -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class BackupException extends HBaseIOException { - private BackupInfo description; - - /** - * Some exception happened for a backup and don't even know the backup that it was about - * @param msg Full description of the failure - */ - public BackupException(String msg) { - super(msg); - } - - /** - * Some exception happened for a backup with a cause - * @param cause the cause - */ - public BackupException(Throwable cause) { - super(cause); - } - - /** - * Exception for the given backup that has no previous root cause - * @param msg reason why the backup failed - * @param desc description of the backup that is being failed - */ - public BackupException(String msg, BackupInfo desc) { - super(msg); - this.description = desc; - } - - /** - * Exception for the given backup due to another exception - * @param msg reason why the backup failed - * @param cause root cause of the failure - * @param desc description of the backup that is being failed - */ - public BackupException(String msg, Throwable cause, BackupInfo desc) { - super(msg, cause); - this.description = desc; - } - - /** - * Exception when the description of the backup cannot be determined, due to some other root - * cause - * @param message description of what caused the failure - * @param e root cause - */ - public BackupException(String message, Exception e) { - super(message, e); - } - - public BackupInfo getBackupContext() { - return this.description; - } - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java index 05f8eb5..007b226 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java @@ -39,13 +39,13 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo; import org.apache.hadoop.hbase.backup.BackupType; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.util.FSUtils; import com.google.protobuf.InvalidProtocolBufferException; @@ -322,7 +322,7 @@ public class BackupManifest { try { FileSystem fs = backupPath.getFileSystem(conf); - FileStatus[] subFiles = FSUtils.listStatus(fs, backupPath); + FileStatus[] subFiles = BackupClientUtil.listStatus(fs, backupPath, null); if (subFiles == null) { String errorMsg = backupPath.toString() + " does not exist"; LOG.error(errorMsg); @@ -364,7 +364,7 @@ public class BackupManifest { loadDependency(proto); //TODO: merge will be implemented by future jira LOG.debug("Loaded manifest instance from manifest file: " - + FSUtils.getPath(subFile.getPath())); + + BackupClientUtil.getPath(subFile.getPath())); return; } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupRestoreConstants.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupRestoreConstants.java deleted file mode 100644 index 7233bfa..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupRestoreConstants.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -/** - * HConstants holds a bunch of HBase Backup and Restore constants - */ -@InterfaceAudience.Private -@InterfaceStability.Stable -public final class BackupRestoreConstants { - - - // delimiter in tablename list in restore command - public static final String TABLENAME_DELIMITER_IN_COMMAND = ","; - - public static final String CONF_STAGING_ROOT = "snapshot.export.staging.root"; - - public static final String BACKUPID_PREFIX = "backup_"; - - public static enum BackupCommand { - CREATE, CANCEL, DELETE, DESCRIBE, HISTORY, STATUS, CONVERT, MERGE, STOP, SHOW, HELP, PROGRESS, SET, - SET_ADD, SET_REMOVE, SET_DELETE, SET_DESCRIBE, SET_LIST - } - - private BackupRestoreConstants() { - // Can't be instantiated with this ctor. - } -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java deleted file mode 100644 index 8c3c2be..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ /dev/null @@ -1,802 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.impl; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; - -/** - * This class provides 'hbase:backup' table API - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public final class BackupSystemTable implements Closeable { - - static class WALItem { - String backupId; - String walFile; - String backupRoot; - - WALItem(String backupId, String walFile, String backupRoot) - { - this.backupId = backupId; - this.walFile = walFile; - this.backupRoot = backupRoot; - } - - public String getBackupId() { - return backupId; - } - - public String getWalFile() { - return walFile; - } - - public String getBackupRoot() { - return backupRoot; - } - - public String toString() { - return backupRoot+"/"+backupId + "/" + walFile; - } - - } - - private static final Log LOG = LogFactory.getLog(BackupSystemTable.class); - private final static TableName tableName = TableName.BACKUP_TABLE_NAME; - // Stores backup sessions (contexts) - final static byte[] SESSIONS_FAMILY = "session".getBytes(); - // Stores other meta - final static byte[] META_FAMILY = "meta".getBytes(); - // Connection to HBase cluster, shared - // among all instances - private final Connection connection; - - public BackupSystemTable(Connection conn) throws IOException { - this.connection = conn; - } - - - public void close() { - // do nothing - } - - /** - * Updates status (state) of a backup session in hbase:backup table - * @param context context - * @throws IOException exception - */ - public void updateBackupInfo(BackupInfo context) throws IOException { - - if (LOG.isDebugEnabled()) { - LOG.debug("update backup status in hbase:backup for: " + context.getBackupId() - + " set status=" + context.getState()); - } - try (Table table = connection.getTable(tableName)) { - Put put = BackupSystemTableHelper.createPutForBackupContext(context); - table.put(put); - } - } - - /** - * Deletes backup status from hbase:backup table - * @param backupId backup id - * @return true, if operation succeeded, false - otherwise - * @throws IOException exception - */ - - public void deleteBackupInfo(String backupId) throws IOException { - - if (LOG.isDebugEnabled()) { - LOG.debug("delete backup status in hbase:backup for " + backupId); - } - try (Table table = connection.getTable(tableName)) { - Delete del = BackupSystemTableHelper.createDeleteForBackupInfo(backupId); - table.delete(del); - } - } - - /** - * Reads backup status object (instance of BackupContext) from hbase:backup table - * @param backupId - backupId - * @return Current status of backup session or null - */ - - public BackupInfo readBackupInfo(String backupId) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("read backup status from hbase:backup for: " + backupId); - } - - try (Table table = connection.getTable(tableName)) { - Get get = BackupSystemTableHelper.createGetForBackupContext(backupId); - Result res = table.get(get); - if(res.isEmpty()){ - return null; - } - return BackupSystemTableHelper.resultToBackupInfo(res); - } - } - - /** - * Read the last backup start code (timestamp) of last successful backup. Will return null if - * there is no start code stored on hbase or the value is of length 0. These two cases indicate - * there is no successful backup completed so far. - * @param backupRoot root directory path to backup - * @return the timestamp of last successful backup - * @throws IOException exception - */ - public String readBackupStartCode(String backupRoot) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("read backup start code from hbase:backup"); - } - try (Table table = connection.getTable(tableName)) { - Get get = BackupSystemTableHelper.createGetForStartCode(backupRoot); - Result res = table.get(get); - if (res.isEmpty()) { - return null; - } - Cell cell = res.listCells().get(0); - byte[] val = CellUtil.cloneValue(cell); - if (val.length == 0){ - return null; - } - return new String(val); - } - } - - /** - * Write the start code (timestamp) to hbase:backup. If passed in null, then write 0 byte. - * @param startCode start code - * @throws IOException exception - */ - public void writeBackupStartCode(Long startCode, String backupRoot) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("write backup start code to hbase:backup " + startCode); - } - try (Table table = connection.getTable(tableName)) { - Put put = BackupSystemTableHelper.createPutForStartCode(startCode.toString(), backupRoot); - table.put(put); - } - } - - /** - * Get the Region Servers log information after the last log roll from hbase:backup. - * @return RS log info - * @throws IOException exception - */ - public HashMap readRegionServerLastLogRollResult(String backupRoot) - throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("read region server last roll log result to hbase:backup"); - } - - Scan scan = BackupSystemTableHelper.createScanForReadRegionServerLastLogRollResult(backupRoot); - scan.setMaxVersions(1); - - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - Result res = null; - HashMap rsTimestampMap = new HashMap(); - while ((res = scanner.next()) != null) { - res.advance(); - Cell cell = res.current(); - byte[] row = CellUtil.cloneRow(cell); - String server = - BackupSystemTableHelper.getServerNameForReadRegionServerLastLogRollResult(row); - byte[] data = CellUtil.cloneValue(cell); - rsTimestampMap.put(server, Long.parseLong(new String(data))); - } - return rsTimestampMap; - } - } - - /** - * Writes Region Server last roll log result (timestamp) to hbase:backup table - * @param server - Region Server name - * @param timestamp - last log timestamp - * @throws IOException exception - */ - public void writeRegionServerLastLogRollResult(String server, Long ts, String backupRoot) - throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("write region server last roll log result to hbase:backup"); - } - try (Table table = connection.getTable(tableName)) { - Put put = - BackupSystemTableHelper.createPutForRegionServerLastLogRollResult(server,ts,backupRoot); - table.put(put); - } - } - - /** - * Get all completed backup information (in desc order by time) - * @param onlyCompeleted, true, if only successfully completed sessions - * @return history info of BackupCompleteData - * @throws IOException exception - */ - public ArrayList getBackupHistory(boolean onlyCompleted) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("get backup history from hbase:backup"); - } - Scan scan = BackupSystemTableHelper.createScanForBackupHistory(); - scan.setMaxVersions(1); - - ArrayList list = new ArrayList(); - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - - Result res = null; - while ((res = scanner.next()) != null) { - res.advance(); - BackupInfo context = BackupSystemTableHelper.cellToBackupInfo(res.current()); - if(onlyCompleted && context.getState() != BackupState.COMPLETE){ - continue; - } - list.add(context); - } - return BackupUtil.sortHistoryListDesc(list); - } - } - - public ArrayList getBackupHistory() throws IOException { - return getBackupHistory(false); - } - - /** - * Get all backup session with a given status (in desc order by time) - * @param status status - * @return history info of backup contexts - * @throws IOException exception - */ - public ArrayList getBackupContexts(BackupState status) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("get backup contexts from hbase:backup"); - } - - Scan scan = BackupSystemTableHelper.createScanForBackupHistory(); - scan.setMaxVersions(1); - ArrayList list = new ArrayList(); - - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - Result res = null; - while ((res = scanner.next()) != null) { - res.advance(); - BackupInfo context = BackupSystemTableHelper.cellToBackupInfo(res.current()); - if (context.getState() != status){ - continue; - } - list.add(context); - } - return list; - } - } - - /** - * Write the current timestamps for each regionserver to hbase:backup after a successful full or - * incremental backup. The saved timestamp is of the last log file that was backed up already. - * @param tables tables - * @param newTimestamps timestamps - * @throws IOException exception - */ - public void writeRegionServerLogTimestamp(Set tables, - HashMap newTimestamps, String backupRoot) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("write RS log ts to HBASE_BACKUP"); - } - List puts = new ArrayList(); - for (TableName table : tables) { - byte[] smapData = toTableServerTimestampProto(table, newTimestamps).toByteArray(); - Put put = - BackupSystemTableHelper.createPutForWriteRegionServerLogTimestamp(table, smapData, backupRoot); - puts.add(put); - } - try (Table table = connection.getTable(tableName)) { - table.put(puts); - } - } - - /** - * Read the timestamp for each region server log after the last successful backup. Each table has - * its own set of the timestamps. The info is stored for each table as a concatenated string of - * rs->timestapmp - * @return the timestamp for each region server. key: tableName value: - * RegionServer,PreviousTimeStamp - * @throws IOException exception - */ - public HashMap> readLogTimestampMap(String backupRoot) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("read RS log ts from HBASE_BACKUP"); - } - - HashMap> tableTimestampMap = - new HashMap>(); - - Scan scan = BackupSystemTableHelper.createScanForReadLogTimestampMap(backupRoot); - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - Result res = null; - while ((res = scanner.next()) != null) { - res.advance(); - Cell cell = res.current(); - byte[] row = CellUtil.cloneRow(cell); - String tabName = BackupSystemTableHelper.getTableNameForReadLogTimestampMap(row); - TableName tn = TableName.valueOf(tabName); - HashMap lastBackup = new HashMap(); - byte[] data = CellUtil.cloneValue(cell); - - if (data == null) { - throw new IOException("Data of last backup data from HBASE_BACKUP " - + "is empty. Create a backup first."); - } - if (data != null && data.length > 0) { - lastBackup = - fromTableServerTimestampProto(BackupProtos.TableServerTimestamp.parseFrom(data)); - tableTimestampMap.put(tn, lastBackup); - } - } - return tableTimestampMap; - } - } - - private BackupProtos.TableServerTimestamp toTableServerTimestampProto(TableName table, - Map map) { - BackupProtos.TableServerTimestamp.Builder tstBuilder = - BackupProtos.TableServerTimestamp.newBuilder(); - tstBuilder.setTable(ProtobufUtil.toProtoTableName(table)); - - for(Entry entry: map.entrySet()) { - BackupProtos.ServerTimestamp.Builder builder = BackupProtos.ServerTimestamp.newBuilder(); - builder.setServer(entry.getKey()); - builder.setTimestamp(entry.getValue()); - tstBuilder.addServerTimestamp(builder.build()); - } - - return tstBuilder.build(); - } - - private HashMap fromTableServerTimestampProto( - BackupProtos.TableServerTimestamp proto) { - HashMap map = new HashMap (); - List list = proto.getServerTimestampList(); - for(BackupProtos.ServerTimestamp st: list) { - map.put(st.getServer(), st.getTimestamp()); - } - return map; - } - - /** - * Return the current tables covered by incremental backup. - * @return set of tableNames - * @throws IOException exception - */ - public Set getIncrementalBackupTableSet(String backupRoot) - throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("get incr backup table set from hbase:backup"); - } - TreeSet set = new TreeSet<>(); - - try (Table table = connection.getTable(tableName)) { - Get get = BackupSystemTableHelper.createGetForIncrBackupTableSet(backupRoot); - Result res = table.get(get); - if (res.isEmpty()) { - return set; - } - List cells = res.listCells(); - for (Cell cell : cells) { - // qualifier = table name - we use table names as qualifiers - set.add(TableName.valueOf(CellUtil.cloneQualifier(cell))); - } - return set; - } - } - - /** - * Add tables to global incremental backup set - * @param tables - set of tables - * @throws IOException exception - */ - public void addIncrementalBackupTableSet(Set tables, String backupRoot) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("add incr backup table set to hbase:backup"); - for (TableName table : tables) { - LOG.debug(table); - } - } - try (Table table = connection.getTable(tableName)) { - Put put = BackupSystemTableHelper.createPutForIncrBackupTableSet(tables, backupRoot); - table.put(put); - } - } - - /** - * Register WAL files as eligible for deletion - * @param files files - * @throws IOException exception - */ - public void addWALFiles(List files, String backupId, - String backupRoot) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("add WAL files to hbase:backup: "+backupId +" "+backupRoot); - for(String f: files){ - LOG.debug("add :"+f); - } - } - try (Table table = connection.getTable(tableName)) { - List puts = - BackupSystemTableHelper.createPutsForAddWALFiles(files, backupId, backupRoot); - table.put(puts); - } - } - - /** - * Register WAL files as eligible for deletion - * @param files files - * @throws IOException exception - */ - public Iterator getWALFilesIterator(String backupRoot) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("get WAL files from hbase:backup"); - } - final Table table = connection.getTable(tableName); - Scan scan = BackupSystemTableHelper.createScanForGetWALs(backupRoot); - final ResultScanner scanner = table.getScanner(scan); - final Iterator it = scanner.iterator(); - return new Iterator() { - - @Override - public boolean hasNext() { - boolean next = it.hasNext(); - if (!next) { - // close all - try { - scanner.close(); - table.close(); - } catch (Exception e) { - LOG.error(e); - } - } - return next; - } - - @Override - public WALItem next() { - Result next = it.next(); - List cells = next.listCells(); - byte[] buf = cells.get(0).getValueArray(); - int len = cells.get(0).getValueLength(); - int offset = cells.get(0).getValueOffset(); - String backupId = new String(buf, offset, len); - buf = cells.get(1).getValueArray(); - len = cells.get(1).getValueLength(); - offset = cells.get(1).getValueOffset(); - String walFile = new String(buf, offset, len); - buf = cells.get(2).getValueArray(); - len = cells.get(2).getValueLength(); - offset = cells.get(2).getValueOffset(); - String backupRoot = new String(buf, offset, len); - return new WALItem(backupId, walFile, backupRoot); - } - - @Override - public void remove() { - // not implemented - throw new RuntimeException("remove is not supported"); - } - }; - - } - - /** - * Check if WAL file is eligible for deletion - * Future: to support all backup destinations - * @param file file - * @return true, if - yes. - * @throws IOException exception - */ - public boolean checkWALFile(String file) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Check if WAL file has been already backuped in hbase:backup "+ file); - } - try (Table table = connection.getTable(tableName)) { - Get get = BackupSystemTableHelper.createGetForCheckWALFile(file); - Result res = table.get(get); - if (res.isEmpty()){ - return false; - } - return true; - } - } - - /** - * Checks if we have at least one backup session in hbase:backup This API is used by - * BackupLogCleaner - * @return true, if - at least one session exists in hbase:backup table - * @throws IOException exception - */ - public boolean hasBackupSessions() throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("has backup sessions from hbase:backup"); - } - boolean result = false; - Scan scan = BackupSystemTableHelper.createScanForBackupHistory(); - scan.setMaxVersions(1); - scan.setCaching(1); - try (Table table = connection.getTable(tableName); - ResultScanner scanner = table.getScanner(scan)) { - if (scanner.next() != null) { - result = true; - } - return result; - } - } - - /** - * BACKUP SETS - */ - - /** - * Get backup set list - * @return backup set list - * @throws IOException - */ - public List listBackupSets() throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug(" backup set list"); - } - List list = new ArrayList(); - Table table = null; - ResultScanner scanner = null; - try { - table = connection.getTable(tableName); - Scan scan = BackupSystemTableHelper.createScanForBackupSetList(); - scan.setMaxVersions(1); - scanner = table.getScanner(scan); - Result res = null; - while ((res = scanner.next()) != null) { - res.advance(); - list.add(BackupSystemTableHelper.cellKeyToBackupSetName(res.current())); - } - return list; - } finally { - if (table != null) { - table.close(); - } - } - } - - /** - * Get backup set description (list of tables) - * @param setName set's name - * @return list of tables in a backup set - * @throws IOException - */ - public List describeBackupSet(String name) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug(" backup set describe: "+name); - } - Table table = null; - try { - table = connection.getTable(tableName); - Get get = BackupSystemTableHelper.createGetForBackupSet(name); - Result res = table.get(get); - if(res.isEmpty()) return new ArrayList(); - res.advance(); - String[] tables = - BackupSystemTableHelper.cellValueToBackupSet(res.current()); - return toList(tables); - } finally { - if (table != null) { - table.close(); - } - } - } - - private List toList(String[] tables) - { - List list = new ArrayList(tables.length); - for(String name: tables) { - list.add(TableName.valueOf(name)); - } - return list; - } - - /** - * Add backup set (list of tables) - * @param name - set name - * @param tables - list of tables, comma-separated - * @throws IOException - */ - public void addToBackupSet(String name, String[] newTables) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug(" backup set add: "+name); - } - Table table = null; - String[] union = null; - try { - table = connection.getTable(tableName); - Get get = BackupSystemTableHelper.createGetForBackupSet(name); - Result res = table.get(get); - if(res.isEmpty()) { - union = newTables; - } else { - res.advance(); - String[] tables = - BackupSystemTableHelper.cellValueToBackupSet(res.current()); - union = merge(tables, newTables); - } - Put put = BackupSystemTableHelper.createPutForBackupSet(name, union); - table.put(put); - } finally { - if (table != null) { - table.close(); - } - } - } - - private String[] merge(String[] tables, String[] newTables) { - List list = new ArrayList(); - // Add all from tables - for(String t: tables){ - list.add(t); - } - for(String nt: newTables){ - if(list.contains(nt)) continue; - list.add(nt); - } - String[] arr = new String[list.size()]; - list.toArray(arr); - return arr; - } - - /** - * Remove tables from backup set (list of tables) - * @param name - set name - * @param tables - list of tables, comma-separated - * @throws IOException - */ - public void removeFromBackupSet(String name, String[] toRemove) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug(" backup set describe: " + name); - } - Table table = null; - String[] disjoint = null; - try { - table = connection.getTable(tableName); - Get get = BackupSystemTableHelper.createGetForBackupSet(name); - Result res = table.get(get); - if (res.isEmpty()) { - return; - } else { - res.advance(); - String[] tables = BackupSystemTableHelper.cellValueToBackupSet(res.current()); - disjoint = disjoin(tables, toRemove); - } - if (disjoint.length > 0) { - Put put = BackupSystemTableHelper.createPutForBackupSet(name, disjoint); - table.put(put); - } else { - // Delete - describeBackupSet(name); - } - } finally { - if (table != null) { - table.close(); - } - } - } - - private String[] disjoin(String[] tables, String[] toRemove) { - List list = new ArrayList(); - // Add all from tables - for (String t : tables) { - list.add(t); - } - for (String nt : toRemove) { - if (list.contains(nt)) { - list.remove(nt); - } - } - String[] arr = new String[list.size()]; - list.toArray(arr); - return arr; - } - - /** - * Delete backup set - * @param name set's name - * @throws IOException - */ - public void deleteBackupSet(String name) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug(" backup set delete: " + name); - } - Table table = null; - try { - table = connection.getTable(tableName); - Delete del = BackupSystemTableHelper.createDeleteForBackupSet(name); - table.delete(del); - } finally { - if (table != null) { - table.close(); - } - } - } - - /** - * Get backup system table descriptor - * @return descriptor - */ - public static HTableDescriptor getSystemTableDescriptor() { - HTableDescriptor tableDesc = new HTableDescriptor(tableName); - HColumnDescriptor colSessionsDesc = new HColumnDescriptor(SESSIONS_FAMILY); - colSessionsDesc.setMaxVersions(1); - // Time to keep backup sessions (secs) - Configuration config = HBaseConfiguration.create(); - int ttl = - config.getInt(HConstants.BACKUP_SYSTEM_TTL_KEY, HConstants.BACKUP_SYSTEM_TTL_DEFAULT); - colSessionsDesc.setTimeToLive(ttl); - tableDesc.addFamily(colSessionsDesc); - HColumnDescriptor colMetaDesc = new HColumnDescriptor(META_FAMILY); - //colDesc.setMaxVersions(1); - tableDesc.addFamily(colMetaDesc); - return tableDesc; - } - - public static String getTableNameAsString() { - return tableName.getNameAsString(); - } - - public static TableName getTableName() { - return tableName; - } -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java deleted file mode 100644 index 7b82655..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTableHelper.java +++ /dev/null @@ -1,423 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.util.Bytes; - - -/** - * A collection for methods used by BackupSystemTable. - */ - -@InterfaceAudience.Private -@InterfaceStability.Evolving -public final class BackupSystemTableHelper { - - /** - * hbase:backup schema: - * 1. Backup sessions rowkey= "session:" + backupId; value = serialized - * BackupContext - * 2. Backup start code rowkey = "startcode:" + backupRoot; value = startcode - * 3. Incremental backup set rowkey="incrbackupset:" + backupRoot; value=[list of tables] - * 4. Table-RS-timestamp map rowkey="trslm:"+ backupRoot+table_name; value = map[RS-> - * last WAL timestamp] - * 5. RS - WAL ts map rowkey="rslogts:"+backupRoot +server; value = last WAL timestamp - * 6. WALs recorded rowkey="wals:"+WAL unique file name; value = backupId and full WAL file name - */ - - private final static String BACKUP_INFO_PREFIX = "session:"; - private final static String START_CODE_ROW = "startcode:"; - private final static String INCR_BACKUP_SET = "incrbackupset:"; - private final static String TABLE_RS_LOG_MAP_PREFIX = "trslm:"; - private final static String RS_LOG_TS_PREFIX = "rslogts:"; - private final static String WALS_PREFIX = "wals:"; - private final static String SET_KEY_PREFIX = "backupset:"; - - private final static byte[] EMPTY_VALUE = new byte[] {}; - - // Safe delimiter in a string - private final static String NULL = "\u0000"; - - private BackupSystemTableHelper() { - throw new AssertionError("Instantiating utility class..."); - } - - /** - * Creates Put operation for a given backup context object - * @param context backup context - * @return put operation - * @throws IOException exception - */ - static Put createPutForBackupContext(BackupInfo context) throws IOException { - Put put = new Put(rowkey(BACKUP_INFO_PREFIX, context.getBackupId())); - put.addColumn(BackupSystemTable.SESSIONS_FAMILY, "context".getBytes(), context.toByteArray()); - return put; - } - - /** - * Creates Get operation for a given backup id - * @param backupId - backup's ID - * @return get operation - * @throws IOException exception - */ - static Get createGetForBackupContext(String backupId) throws IOException { - Get get = new Get(rowkey(BACKUP_INFO_PREFIX, backupId)); - get.addFamily(BackupSystemTable.SESSIONS_FAMILY); - get.setMaxVersions(1); - return get; - } - - /** - * Creates Delete operation for a given backup id - * @param backupId - backup's ID - * @return delete operation - * @throws IOException exception - */ - public static Delete createDeleteForBackupInfo(String backupId) { - Delete del = new Delete(rowkey(BACKUP_INFO_PREFIX, backupId)); - del.addFamily(BackupSystemTable.SESSIONS_FAMILY); - return del; - } - - /** - * Converts Result to BackupContext - * @param res - HBase result - * @return backup context instance - * @throws IOException exception - */ - static BackupInfo resultToBackupInfo(Result res) throws IOException { - res.advance(); - Cell cell = res.current(); - return cellToBackupInfo(cell); - } - - /** - * Creates Get operation to retrieve start code from hbase:backup - * @return get operation - * @throws IOException exception - */ - static Get createGetForStartCode(String rootPath) throws IOException { - Get get = new Get(rowkey(START_CODE_ROW, rootPath)); - get.addFamily(BackupSystemTable.META_FAMILY); - get.setMaxVersions(1); - return get; - } - - /** - * Creates Put operation to store start code to hbase:backup - * @return put operation - * @throws IOException exception - */ - static Put createPutForStartCode(String startCode, String rootPath) { - Put put = new Put(rowkey(START_CODE_ROW, rootPath)); - put.addColumn(BackupSystemTable.META_FAMILY, "startcode".getBytes(), startCode.getBytes()); - return put; - } - - /** - * Creates Get to retrieve incremental backup table set from hbase:backup - * @return get operation - * @throws IOException exception - */ - static Get createGetForIncrBackupTableSet(String backupRoot) throws IOException { - Get get = new Get(rowkey(INCR_BACKUP_SET, backupRoot)); - get.addFamily(BackupSystemTable.META_FAMILY); - get.setMaxVersions(1); - return get; - } - - /** - * Creates Put to store incremental backup table set - * @param tables tables - * @return put operation - */ - static Put createPutForIncrBackupTableSet(Set tables, String backupRoot) { - Put put = new Put(rowkey(INCR_BACKUP_SET, backupRoot)); - for (TableName table : tables) { - put.addColumn(BackupSystemTable.META_FAMILY, Bytes.toBytes(table.getNameAsString()), - EMPTY_VALUE); - } - return put; - } - - /** - * Creates Scan operation to load backup history - * @return scan operation - */ - static Scan createScanForBackupHistory() { - Scan scan = new Scan(); - byte[] startRow = BACKUP_INFO_PREFIX.getBytes(); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.SESSIONS_FAMILY); - - return scan; - } - - /** - * Converts cell to backup context instance. - * @param current - cell - * @return backup context instance - * @throws IOException exception - */ - static BackupInfo cellToBackupInfo(Cell current) throws IOException { - byte[] data = CellUtil.cloneValue(current); - return BackupInfo.fromByteArray(data); - } - - /** - * Creates Put to write RS last roll log timestamp map - * @param table - table - * @param smap - map, containing RS:ts - * @return put operation - */ - static Put createPutForWriteRegionServerLogTimestamp(TableName table, byte[] smap, - String backupRoot) { - Put put = new Put(rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot, NULL, table.getNameAsString())); - put.addColumn(BackupSystemTable.META_FAMILY, "log-roll-map".getBytes(), smap); - return put; - } - - /** - * Creates Scan to load table-> { RS -> ts} map of maps - * @return scan operation - */ - static Scan createScanForReadLogTimestampMap(String backupRoot) { - Scan scan = new Scan(); - byte[] startRow = rowkey(TABLE_RS_LOG_MAP_PREFIX, backupRoot); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.META_FAMILY); - - return scan; - } - - /** - * Get table name from rowkey - * @param cloneRow rowkey - * @return table name - */ - static String getTableNameForReadLogTimestampMap(byte[] cloneRow) { - String s = new String(cloneRow); - int index = s.lastIndexOf(NULL); - return s.substring(index +1); - } - - /** - * Creates Put to store RS last log result - * @param server - server name - * @param timestamp - log roll result (timestamp) - * @return put operation - */ - static Put createPutForRegionServerLastLogRollResult(String server, - Long timestamp, String backupRoot ) { - Put put = new Put(rowkey(RS_LOG_TS_PREFIX, backupRoot, NULL, server)); - put.addColumn(BackupSystemTable.META_FAMILY, "rs-log-ts".getBytes(), - timestamp.toString().getBytes()); - return put; - } - - /** - * Creates Scan operation to load last RS log roll results - * @return scan operation - */ - static Scan createScanForReadRegionServerLastLogRollResult(String backupRoot) { - Scan scan = new Scan(); - byte[] startRow = rowkey(RS_LOG_TS_PREFIX, backupRoot); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.META_FAMILY); - - return scan; - } - - /** - * Get server's name from rowkey - * @param row - rowkey - * @return server's name - */ - static String getServerNameForReadRegionServerLastLogRollResult(byte[] row) { - String s = new String(row); - int index = s.lastIndexOf(NULL); - return s.substring(index +1); - } - - /** - * Creates put list for list of WAL files - * @param files list of WAL file paths - * @param backupId backup id - * @return put list - * @throws IOException exception - */ - public static List createPutsForAddWALFiles(List files, - String backupId, String backupRoot) - throws IOException { - - List puts = new ArrayList(); - for (String file : files) { - Put put = new Put(rowkey(WALS_PREFIX, BackupUtil.getUniqueWALFileNamePart(file))); - put.addColumn(BackupSystemTable.META_FAMILY, "backupId".getBytes(), backupId.getBytes()); - put.addColumn(BackupSystemTable.META_FAMILY, "file".getBytes(), file.getBytes()); - put.addColumn(BackupSystemTable.META_FAMILY, "root".getBytes(), backupRoot.getBytes()); - puts.add(put); - } - return puts; - } - - /** - * Creates Scan operation to load WALs - * TODO: support for backupRoot - * @param backupRoot - path to backup destination - * @return scan operation - */ - public static Scan createScanForGetWALs(String backupRoot) { - Scan scan = new Scan(); - byte[] startRow = WALS_PREFIX.getBytes(); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.META_FAMILY); - return scan; - } - /** - * Creates Get operation for a given wal file name - * TODO: support for backup destination - * @param file file - * @return get operation - * @throws IOException exception - */ - public static Get createGetForCheckWALFile(String file) throws IOException { - Get get = new Get(rowkey(WALS_PREFIX, BackupUtil.getUniqueWALFileNamePart(file))); - // add backup root column - get.addFamily(BackupSystemTable.META_FAMILY); - return get; - } - - - /** - * Creates Scan operation to load backup set list - * @return scan operation - */ - static Scan createScanForBackupSetList() { - Scan scan = new Scan(); - byte[] startRow = SET_KEY_PREFIX.getBytes(); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.setStartRow(startRow); - scan.setStopRow(stopRow); - scan.addFamily(BackupSystemTable.META_FAMILY); - return scan; - } - - /** - * Creates Get operation to load backup set content - * @return get operation - */ - static Get createGetForBackupSet(String name) { - Get get = new Get(rowkey(SET_KEY_PREFIX, name)); - get.addFamily(BackupSystemTable.META_FAMILY); - return get; - } - - /** - * Creates Delete operation to delete backup set content - * @return delete operation - */ - static Delete createDeleteForBackupSet(String name) { - Delete del = new Delete(rowkey(SET_KEY_PREFIX, name)); - del.addFamily(BackupSystemTable.META_FAMILY); - return del; - } - - - /** - * Creates Put operation to update backup set content - * @return put operation - */ - static Put createPutForBackupSet(String name, String[] tables) { - Put put = new Put(rowkey(SET_KEY_PREFIX, name)); - byte[] value = convertToByteArray(tables); - put.addColumn(BackupSystemTable.META_FAMILY, "tables".getBytes(), value); - return put; - } - - private static byte[] convertToByteArray(String[] tables) { - return StringUtils.join(tables, ",").getBytes(); - } - - - /** - * Converts cell to backup set list. - * @param current - cell - * @return backup set - * @throws IOException - */ - static String[] cellValueToBackupSet(Cell current) throws IOException { - byte[] data = CellUtil.cloneValue(current); - if( data != null && data.length > 0){ - return new String(data).split(","); - } else{ - return new String[0]; - } - } - - /** - * Converts cell key to backup set name. - * @param current - cell - * @return backup set name - * @throws IOException - */ - static String cellKeyToBackupSetName(Cell current) throws IOException { - byte[] data = CellUtil.cloneRow(current); - return new String(data).substring(SET_KEY_PREFIX.length()); - } - - static byte[] rowkey(String s, String ... other){ - StringBuilder sb = new StringBuilder(s); - for(String ss: other){ - sb.append(ss); - } - return sb.toString().getBytes(); - } - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupUtil.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupUtil.java deleted file mode 100644 index df2b0a6..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupUtil.java +++ /dev/null @@ -1,472 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.io.InterruptedIOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; -import java.util.TreeMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupClientUtil; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; -import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSTableDescriptors; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.wal.DefaultWALProvider; - -/** - * A collection for methods used by multiple classes to backup HBase tables. - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public final class BackupUtil { - protected static final Log LOG = LogFactory.getLog(BackupUtil.class); - public static final String LOGNAME_SEPARATOR = "."; - - private BackupUtil(){ - throw new AssertionError("Instantiating utility class..."); - } - - public static void waitForSnapshot(SnapshotDescription snapshot, long max, - SnapshotManager snapshotMgr, Configuration conf) throws IOException { - boolean done = false; - long start = EnvironmentEdgeManager.currentTime(); - int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); - long maxPauseTime = max / numRetries; - int tries = 0; - LOG.debug("Waiting a max of " + max + " ms for snapshot '" + - ClientSnapshotDescriptionUtils.toString(snapshot) + "'' to complete. (max " + - maxPauseTime + " ms per retry)"); - while (tries == 0 - || ((EnvironmentEdgeManager.currentTime() - start) < max && !done)) { - try { - // sleep a backoff <= pauseTime amount - long pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, - HConstants.DEFAULT_HBASE_CLIENT_PAUSE); - long sleep = HBaseAdmin.getPauseTime(tries++, pause); - sleep = sleep > maxPauseTime ? maxPauseTime : sleep; - LOG.debug("(#" + tries + ") Sleeping: " + sleep + - "ms while waiting for snapshot completion."); - Thread.sleep(sleep); - } catch (InterruptedException e) { - throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e); - } - LOG.debug("Getting current status of snapshot ..."); - done = snapshotMgr.isSnapshotDone(snapshot); - } - if (!done) { - throw new SnapshotCreationException("Snapshot '" + snapshot.getName() - + "' wasn't completed in expectedTime:" + max + " ms", snapshot); - } - } - - /** - * Loop through the RS log timestamp map for the tables, for each RS, find the min timestamp - * value for the RS among the tables. - * @param rsLogTimestampMap timestamp map - * @return the min timestamp of each RS - */ - public static HashMap getRSLogTimestampMins( - HashMap> rsLogTimestampMap) { - - if (rsLogTimestampMap == null || rsLogTimestampMap.isEmpty()) { - return null; - } - - HashMap rsLogTimestamptMins = new HashMap(); - HashMap> rsLogTimestampMapByRS = - new HashMap>(); - - for (Entry> tableEntry : rsLogTimestampMap.entrySet()) { - TableName table = tableEntry.getKey(); - HashMap rsLogTimestamp = tableEntry.getValue(); - for (Entry rsEntry : rsLogTimestamp.entrySet()) { - String rs = rsEntry.getKey(); - Long ts = rsEntry.getValue(); - if (!rsLogTimestampMapByRS.containsKey(rs)) { - rsLogTimestampMapByRS.put(rs, new HashMap()); - rsLogTimestampMapByRS.get(rs).put(table, ts); - } else { - rsLogTimestampMapByRS.get(rs).put(table, ts); - } - } - } - - for (String rs : rsLogTimestampMapByRS.keySet()) { - rsLogTimestamptMins.put(rs, BackupClientUtil.getMinValue(rsLogTimestampMapByRS.get(rs))); - } - - return rsLogTimestamptMins; - } - - /** - * copy out Table RegionInfo into incremental backup image need to consider move this logic into - * HBackupFileSystem - * @param backupContext backup context - * @param conf configuration - * @throws IOException exception - * @throws InterruptedException exception - */ - public static void copyTableRegionInfo(BackupInfo backupContext, Configuration conf) - throws IOException, InterruptedException { - - Path rootDir = FSUtils.getRootDir(conf); - FileSystem fs = rootDir.getFileSystem(conf); - - // for each table in the table set, copy out the table info and region info files in the correct - // directory structure - for (TableName table : backupContext.getTables()) { - - LOG.debug("Attempting to copy table info for:" + table); - TableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, table); - - // write a copy of descriptor to the target directory - Path target = new Path(backupContext.getBackupStatus(table).getTargetDir()); - FileSystem targetFs = target.getFileSystem(conf); - FSTableDescriptors descriptors = - new FSTableDescriptors(conf, targetFs, FSUtils.getRootDir(conf)); - descriptors.createTableDescriptorForTableDirectory(target, orig, false); - LOG.debug("Finished copying tableinfo."); - - // TODO: optimize - List regions = null; - try(Connection conn = ConnectionFactory.createConnection(conf); - Admin admin = conn.getAdmin()) { - regions = admin.getTableRegions(table); - } catch (Exception e) { - throw new BackupException(e); - } - - // For each region, write the region info to disk - LOG.debug("Starting to write region info for table " + table); - for (HRegionInfo regionInfo : regions) { - Path regionDir = - HRegion.getRegionDir(new Path(backupContext.getBackupStatus(table).getTargetDir()), - regionInfo); - regionDir = - new Path(backupContext.getBackupStatus(table).getTargetDir(), regionDir.getName()); - writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo); - } - LOG.debug("Finished writing region info for table " + table); - } - } - - /** - * Write the .regioninfo file on-disk. - */ - public static void writeRegioninfoOnFilesystem(final Configuration conf, final FileSystem fs, - final Path regionInfoDir, HRegionInfo regionInfo) throws IOException { - final byte[] content = regionInfo.toDelimitedByteArray(); - Path regionInfoFile = new Path(regionInfoDir, ".regioninfo"); - // First check to get the permissions - FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY); - // Write the RegionInfo file content - FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null); - try { - out.write(content); - } finally { - out.close(); - } - } - - /** - * TODO: return hostname:port - * @param p - * @return host name: port - * @throws IOException - */ - public static String parseHostNameFromLogFile(Path p) throws IOException { - if (isArchivedLogFile(p)) { - return BackupClientUtil.parseHostFromOldLog(p); - } else { - ServerName sname = DefaultWALProvider.getServerNameFromWALDirectoryName(p); - return sname.getHostname() + ":" + sname.getPort(); - } - } - - private static boolean isArchivedLogFile(Path p) { - String oldLog = Path.SEPARATOR + HConstants.HREGION_OLDLOGDIR_NAME + Path.SEPARATOR; - return p.toString().contains(oldLog); - } - - /** - * Returns WAL file name - * @param walFileName WAL file name - * @return WAL file name - * @throws IOException exception - * @throws IllegalArgumentException exception - */ - public static String getUniqueWALFileNamePart(String walFileName) throws IOException { - return getUniqueWALFileNamePart(new Path(walFileName)); - } - - /** - * Returns WAL file name - * @param p - WAL file path - * @return WAL file name - * @throws IOException exception - */ - public static String getUniqueWALFileNamePart(Path p) throws IOException { - return p.getName(); - } - - /** - * Get the total length of files under the given directory recursively. - * @param fs The hadoop file system - * @param dir The target directory - * @return the total length of files - * @throws IOException exception - */ - public static long getFilesLength(FileSystem fs, Path dir) throws IOException { - long totalLength = 0; - FileStatus[] files = FSUtils.listStatus(fs, dir); - if (files != null) { - for (FileStatus fileStatus : files) { - if (fileStatus.isDirectory()) { - totalLength += getFilesLength(fs, fileStatus.getPath()); - } else { - totalLength += fileStatus.getLen(); - } - } - } - return totalLength; - } - - - - /** - * Sort history list by start time in descending order. - * @param historyList history list - * @return sorted list of BackupCompleteData - */ - public static ArrayList sortHistoryListDesc( - ArrayList historyList) { - ArrayList list = new ArrayList(); - TreeMap map = new TreeMap(); - for (BackupInfo h : historyList) { - map.put(Long.toString(h.getStartTs()), h); - } - Iterator i = map.descendingKeySet().iterator(); - while (i.hasNext()) { - list.add(map.get(i.next())); - } - return list; - } - - /** - * Get list of all WAL files (WALs and archive) - * @param c - configuration - * @return list of WAL files - * @throws IOException exception - */ - public static List getListOfWALFiles(Configuration c) throws IOException { - Path rootDir = FSUtils.getRootDir(c); - Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); - Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); - List logFiles = new ArrayList(); - - FileSystem fs = FileSystem.get(c); - logFiles = BackupClientUtil.getFiles(fs, logDir, logFiles, null); - logFiles = BackupClientUtil.getFiles(fs, oldLogDir, logFiles, null); - return logFiles; - } - - /** - * Get list of all WAL files (WALs and archive) - * @param c - configuration - * @return list of WAL files - * @throws IOException exception - */ - public static List getListOfWALFiles(Configuration c, PathFilter filter) - throws IOException { - Path rootDir = FSUtils.getRootDir(c); - Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); - Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); - List logFiles = new ArrayList(); - - FileSystem fs = FileSystem.get(c); - logFiles = BackupClientUtil.getFiles(fs, logDir, logFiles, filter); - logFiles = BackupClientUtil.getFiles(fs, oldLogDir, logFiles, filter); - return logFiles; - } - - /** - * Get list of all old WAL files (WALs and archive) - * @param c - configuration - * @return list of WAL files - * @throws IOException exception - */ - public static List getWALFilesOlderThan(final Configuration c, - final HashMap hostTimestampMap) throws IOException { - Path rootDir = FSUtils.getRootDir(c); - Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); - Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); - List logFiles = new ArrayList(); - - PathFilter filter = new PathFilter() { - - @Override - public boolean accept(Path p) { - try { - if (DefaultWALProvider.isMetaFile(p)) { - return false; - } - String host = parseHostNameFromLogFile(p); - Long oldTimestamp = hostTimestampMap.get(host); - Long currentLogTS = BackupClientUtil.getCreationTime(p); - return currentLogTS <= oldTimestamp; - } catch (IOException e) { - LOG.error(e); - return false; - } - } - }; - FileSystem fs = FileSystem.get(c); - logFiles = BackupClientUtil.getFiles(fs, logDir, logFiles, filter); - logFiles = BackupClientUtil.getFiles(fs, oldLogDir, logFiles, filter); - return logFiles; - } - - public static String join(TableName[] names) { - StringBuilder sb = new StringBuilder(); - String sep = BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND; - for (TableName s : names) { - sb.append(sep).append(s.getNameAsString()); - } - return sb.toString(); - } - - public static TableName[] parseTableNames(String tables) { - if (tables == null) { - return null; - } - String[] tableArray = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); - - TableName[] ret = new TableName[tableArray.length]; - for (int i = 0; i < tableArray.length; i++) { - ret[i] = TableName.valueOf(tableArray[i]); - } - return ret; - } - - public static void cleanupBackupData(BackupInfo context, Configuration conf) - throws IOException - { - cleanupHLogDir(context, conf); - cleanupTargetDir(context, conf); - } - - /** - * Clean up directories which are generated when DistCp copying hlogs. - * @throws IOException - */ - private static void cleanupHLogDir(BackupInfo backupContext, Configuration conf) - throws IOException { - - String logDir = backupContext.getHLogTargetDir(); - if (logDir == null) { - LOG.warn("No log directory specified for " + backupContext.getBackupId()); - return; - } - - Path rootPath = new Path(logDir).getParent(); - FileSystem fs = FileSystem.get(rootPath.toUri(), conf); - FileStatus[] files = FSUtils.listStatus(fs, rootPath); - if (files == null) { - return; - } - for (FileStatus file : files) { - LOG.debug("Delete log files: " + file.getPath().getName()); - FSUtils.delete(fs, file.getPath(), true); - } - } - - /** - * Clean up the data at target directory - */ - private static void cleanupTargetDir(BackupInfo backupContext, Configuration conf) { - try { - // clean up the data at target directory - LOG.debug("Trying to cleanup up target dir : " + backupContext.getBackupId()); - String targetDir = backupContext.getTargetRootDir(); - if (targetDir == null) { - LOG.warn("No target directory specified for " + backupContext.getBackupId()); - return; - } - - FileSystem outputFs = - FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf); - - for (TableName table : backupContext.getTables()) { - Path targetDirPath = - new Path(HBackupFileSystem.getTableBackupDir(backupContext.getTargetRootDir(), - backupContext.getBackupId(), table)); - if (outputFs.delete(targetDirPath, true)) { - LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); - } else { - LOG.info("No data has been found in " + targetDirPath.toString() + "."); - } - - Path tableDir = targetDirPath.getParent(); - FileStatus[] backups = FSUtils.listStatus(outputFs, tableDir); - if (backups == null || backups.length == 0) { - outputFs.delete(tableDir, true); - LOG.debug(tableDir.toString() + " is empty, remove it."); - } - } - - } catch (IOException e1) { - LOG.error("Cleaning up backup data of " + backupContext.getBackupId() + " at " - + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + "."); - } - } - - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index 8338fee..8e192f1 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -33,9 +33,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupClientUtil; import org.apache.hadoop.hbase.backup.BackupInfo; import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.backup.util.BackupServerUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Admin; @@ -86,7 +87,7 @@ public class IncrementalBackupManager { HashMap> previousTimestampMap = backupManager.readLogTimestampMap(); - previousTimestampMins = BackupUtil.getRSLogTimestampMins(previousTimestampMap); + previousTimestampMins = BackupServerUtil.getRSLogTimestampMins(previousTimestampMap); if (LOG.isDebugEnabled()) { LOG.debug("StartCode " + savedStartCode + "for backupID " + backupContext.getBackupId()); @@ -170,7 +171,7 @@ public class IncrementalBackupManager { continue; } String walFileName = item.getWalFile(); - String server = BackupUtil.parseHostNameFromLogFile(new Path(walFileName)); + String server = BackupServerUtil.parseHostNameFromLogFile(new Path(walFileName)); Long tss = getTimestamp(walFileName); Long oldTss = olderTimestamps.get(server); Long newTss = newestTimestamps.get(server); @@ -189,7 +190,7 @@ public class IncrementalBackupManager { } private Long getTimestamp(String walFileName) { - int index = walFileName.lastIndexOf(BackupUtil.LOGNAME_SEPARATOR); + int index = walFileName.lastIndexOf(BackupServerUtil.LOGNAME_SEPARATOR); return Long.parseLong(walFileName.substring(index+1)); } @@ -237,7 +238,7 @@ public class IncrementalBackupManager { rss = fs.listStatus(logDir); for (FileStatus rs : rss) { p = rs.getPath(); - host = BackupUtil.parseHostNameFromLogFile(p); + host = BackupServerUtil.parseHostNameFromLogFile(p); FileStatus[] logs; oldTimeStamp = olderTimestamps.get(host); // It is possible that there is no old timestamp in hbase:backup for this host if diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalRestoreService.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalRestoreService.java deleted file mode 100644 index 8904184..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalRestoreService.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -@InterfaceAudience.Private -@InterfaceStability.Evolving -public interface IncrementalRestoreService extends Configurable{ - - /** - * Run restore operation - * @param logDirectoryPaths - path array of WAL log directories - * @param fromTables - from tables - * @param toTables - to tables - * @throws IOException - */ - public void run(Path[] logDirectoryPaths, TableName[] fromTables, TableName[] toTables) - throws IOException; -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreClientImpl.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreClientImpl.java index f16d213..9906f47 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreClientImpl.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreClientImpl.java @@ -34,10 +34,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.BackupClientUtil; import org.apache.hadoop.hbase.backup.HBackupFileSystem; import org.apache.hadoop.hbase.backup.RestoreClient; import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.backup.util.RestoreServerUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Admin; @@ -62,6 +63,8 @@ public final class RestoreClientImpl implements RestoreClient { this.conf = conf; } + + /** * Restore operation. Stage 1: validate backupManifest, and check target tables * @param backupRootDir The root dir for backup image @@ -72,11 +75,10 @@ public final class RestoreClientImpl implements RestoreClient { * @param tTableArray The array of mapping tables to restore to * @param isOverwrite True then do restore overwrite if target table exists, otherwise fail the * request if target table exists - * @return True if only do dependency check * @throws IOException if any failure during restore */ @Override - public boolean restore(String backupRootDir, + public void restore(String backupRootDir, String backupId, boolean check, boolean autoRestore, TableName[] sTableArray, TableName[] tTableArray, boolean isOverwrite) throws IOException { @@ -98,11 +100,6 @@ public final class RestoreClientImpl implements RestoreClient { } } - // return true if only for check - if (check) { - return true; - } - if (tTableArray == null) { tTableArray = sTableArray; } @@ -121,8 +118,6 @@ public final class RestoreClientImpl implements RestoreClient { throw e; } - // not only for check, return false - return false; } @@ -278,7 +273,7 @@ public final class RestoreClientImpl implements RestoreClient { String backupId = image.getBackupId(); Path rootPath = new Path(rootDir); - RestoreUtil restoreTool = new RestoreUtil(conf, rootPath, backupId); + RestoreServerUtil restoreTool = new RestoreServerUtil(conf, rootPath, backupId); BackupManifest manifest = HBackupFileSystem.getManifest(sTable, conf, rootPath, backupId); Path tableBackupPath = HBackupFileSystem.getTableBackupPath(sTable, rootPath, backupId); @@ -320,7 +315,7 @@ public final class RestoreClientImpl implements RestoreClient { Path backupRoot = new Path(rootDir); // We need hFS only for full restore (see the code) - RestoreUtil restoreTool = new RestoreUtil(conf, backupRoot, backupId); + RestoreServerUtil restoreTool = new RestoreServerUtil(conf, backupRoot, backupId); BackupManifest manifest = HBackupFileSystem.getManifest(sTable, conf, backupRoot, backupId); Path tableBackupPath = HBackupFileSystem.getTableBackupPath(sTable, backupRoot, backupId); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreUtil.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreUtil.java deleted file mode 100644 index 592770b..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/RestoreUtil.java +++ /dev/null @@ -1,608 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.TreeMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupRestoreFactory; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.io.HFileLink; -import org.apache.hadoop.hbase.io.hfile.CacheConfig; -import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; -import org.apache.hadoop.hbase.regionserver.HStore; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; -import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; -import org.apache.hadoop.hbase.snapshot.SnapshotManifest; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * A collection for methods used by multiple classes to restore HBase tables. - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class RestoreUtil { - - public static final Log LOG = LogFactory.getLog(RestoreUtil.class); - - private final String[] ignoreDirs = { "recovered.edits" }; - - protected Configuration conf = null; - - protected Path backupRootPath; - - protected String backupId; - - protected FileSystem fs; - private final String RESTORE_TMP_PATH = "/tmp"; - private final Path restoreTmpPath; - - // store table name and snapshot dir mapping - private final HashMap snapshotMap = new HashMap<>(); - - public RestoreUtil(Configuration conf, final Path backupRootPath, final String backupId) - throws IOException { - this.conf = conf; - this.backupRootPath = backupRootPath; - this.backupId = backupId; - this.fs = backupRootPath.getFileSystem(conf); - this.restoreTmpPath = new Path(conf.get("hbase.fs.tmp.dir") != null? - conf.get("hbase.fs.tmp.dir"): RESTORE_TMP_PATH, - "restore"); - } - - /** - * return value represent path for: - * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/archive/data/default/t1_dn" - * @param tabelName table name - * @return path to table archive - * @throws IOException exception - */ - Path getTableArchivePath(TableName tableName) - throws IOException { - Path baseDir = new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, - backupId), HConstants.HFILE_ARCHIVE_DIRECTORY); - Path dataDir = new Path(baseDir, HConstants.BASE_NAMESPACE_DIR); - Path archivePath = new Path(dataDir, tableName.getNamespaceAsString()); - Path tableArchivePath = - new Path(archivePath, tableName.getQualifierAsString()); - if (!fs.exists(tableArchivePath) || !fs.getFileStatus(tableArchivePath).isDirectory()) { - LOG.debug("Folder tableArchivePath: " + tableArchivePath.toString() + " does not exists"); - tableArchivePath = null; // empty table has no archive - } - return tableArchivePath; - } - - /** - * Gets region list - * @param tableName table name - * @return RegionList region list - * @throws FileNotFoundException exception - * @throws IOException exception - */ - ArrayList getRegionList(TableName tableName) - throws FileNotFoundException, IOException { - Path tableArchivePath = this.getTableArchivePath(tableName); - ArrayList regionDirList = new ArrayList(); - FileStatus[] children = fs.listStatus(tableArchivePath); - for (FileStatus childStatus : children) { - // here child refer to each region(Name) - Path child = childStatus.getPath(); - regionDirList.add(child); - } - return regionDirList; - } - - /** - * During incremental backup operation. Call WalPlayer to replay WAL in backup image Currently - * tableNames and newTablesNames only contain single table, will be expanded to multiple tables in - * the future - * @param logDir : incremental backup folders, which contains WAL - * @param tableNames : source tableNames(table names were backuped) - * @param newTableNames : target tableNames(table names to be restored to) - * @throws IOException exception - */ - void incrementalRestoreTable(Path[] logDirs, - TableName[] tableNames, TableName[] newTableNames) throws IOException { - - if (tableNames.length != newTableNames.length) { - throw new IOException("Number of source tables and target tables does not match!"); - } - - // for incremental backup image, expect the table already created either by user or previous - // full backup. Here, check that all new tables exists - try (Connection conn = ConnectionFactory.createConnection(conf); - Admin admin = conn.getAdmin()) { - for (TableName tableName : newTableNames) { - if (!admin.tableExists(tableName)) { - admin.close(); - throw new IOException("HBase table " + tableName - + " does not exist. Create the table first, e.g. by restoring a full backup."); - } - } - IncrementalRestoreService restoreService = - BackupRestoreFactory.getIncrementalRestoreService(conf); - - restoreService.run(logDirs, tableNames, newTableNames); - } - } - - void fullRestoreTable(Path tableBackupPath, TableName tableName, TableName newTableName, - boolean converted) throws IOException { - restoreTableAndCreate(tableName, newTableName, tableBackupPath, converted); - } - - /** - * return value represent path for: - * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/.hbase-snapshot" - * @param backupRootPath backup root path - * @param tableName table name - * @param backupId backup Id - * @return path for snapshot - */ - static Path getTableSnapshotPath(Path backupRootPath, TableName tableName, - String backupId) { - return new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, backupId), - HConstants.SNAPSHOT_DIR_NAME); - } - - /** - * return value represent path for: - * "..../default/t1_dn/backup_1396650096738/.hbase-snapshot/snapshot_1396650097621_default_t1_dn" - * this path contains .snapshotinfo, .tabledesc (0.96 and 0.98) this path contains .snapshotinfo, - * .data.manifest (trunk) - * @param tableName table name - * @return path to table info - * @throws FileNotFoundException exception - * @throws IOException exception - */ - Path getTableInfoPath(TableName tableName) - throws FileNotFoundException, IOException { - Path tableSnapShotPath = getTableSnapshotPath(backupRootPath, tableName, backupId); - Path tableInfoPath = null; - - // can't build the path directly as the timestamp values are different - FileStatus[] snapshots = fs.listStatus(tableSnapShotPath); - for (FileStatus snapshot : snapshots) { - tableInfoPath = snapshot.getPath(); - // SnapshotManifest.DATA_MANIFEST_NAME = "data.manifest"; - if (tableInfoPath.getName().endsWith("data.manifest")) { - break; - } - } - return tableInfoPath; - } - - /** - * @param tableName is the table backed up - * @return {@link HTableDescriptor} saved in backup image of the table - */ - HTableDescriptor getTableDesc(TableName tableName) - throws FileNotFoundException, IOException { - Path tableInfoPath = this.getTableInfoPath(tableName); - SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath); - SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc); - HTableDescriptor tableDescriptor = manifest.getTableDescriptor(); - if (!tableDescriptor.getTableName().equals(tableName)) { - LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: " - + tableInfoPath.toString()); - LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getNameAsString()); - throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName + - " under tableInfoPath: " + tableInfoPath.toString()); - } - return tableDescriptor; - } - - /** - * Duplicate the backup image if it's on local cluster - * @see HStore#bulkLoadHFile(String, long) - * @see HRegionFileSystem#bulkLoadStoreFile(String familyName, Path srcPath, long seqNum) - * @param tableArchivePath archive path - * @return the new tableArchivePath - * @throws IOException exception - */ - Path checkLocalAndBackup(Path tableArchivePath) throws IOException { - // Move the file if it's on local cluster - boolean isCopyNeeded = false; - - FileSystem srcFs = tableArchivePath.getFileSystem(conf); - FileSystem desFs = FileSystem.get(conf); - if (tableArchivePath.getName().startsWith("/")) { - isCopyNeeded = true; - } else { - // This should match what is done in @see HRegionFileSystem#bulkLoadStoreFile(String, Path, - // long) - if (srcFs.getUri().equals(desFs.getUri())) { - LOG.debug("cluster hold the backup image: " + srcFs.getUri() + "; local cluster node: " - + desFs.getUri()); - isCopyNeeded = true; - } - } - if (isCopyNeeded) { - LOG.debug("File " + tableArchivePath + " on local cluster, back it up before restore"); - if (desFs.exists(restoreTmpPath)) { - try { - desFs.delete(restoreTmpPath, true); - } catch (IOException e) { - LOG.debug("Failed to delete path: " + restoreTmpPath - + ", need to check whether restore target DFS cluster is healthy"); - } - } - FileUtil.copy(srcFs, tableArchivePath, desFs, restoreTmpPath, false, conf); - LOG.debug("Copied to temporary path on local cluster: " + restoreTmpPath); - tableArchivePath = restoreTmpPath; - } - return tableArchivePath; - } - - private void restoreTableAndCreate(TableName tableName, TableName newTableName, - Path tableBackupPath, boolean converted) throws IOException { - if (newTableName == null || newTableName.equals("")) { - newTableName = tableName; - } - - FileSystem fileSys = tableBackupPath.getFileSystem(this.conf); - - // get table descriptor first - HTableDescriptor tableDescriptor = null; - - Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId); - - if (fileSys.exists(tableSnapshotPath)) { - // snapshot path exist means the backup path is in HDFS - // check whether snapshot dir already recorded for target table - if (snapshotMap.get(tableName) != null) { - SnapshotDescription desc = - SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath); - SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc); - tableDescriptor = manifest.getTableDescriptor(); - } else { - tableDescriptor = getTableDesc(tableName); - snapshotMap.put(tableName, getTableInfoPath(tableName)); - } - if (tableDescriptor == null) { - LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost"); - } - } else if (converted) { - // first check if this is a converted backup image - LOG.error("convert will be supported in a future jira"); - } - - Path tableArchivePath = getTableArchivePath(tableName); - if (tableArchivePath == null) { - if (tableDescriptor != null) { - // find table descriptor but no archive dir means the table is empty, create table and exit - if(LOG.isDebugEnabled()) { - LOG.debug("find table descriptor but no archive dir for table " + tableName - + ", will only create table"); - } - tableDescriptor.setName(newTableName); - checkAndCreateTable(tableBackupPath, tableName, newTableName, null, tableDescriptor); - return; - } else { - throw new IllegalStateException("Cannot restore hbase table because directory '" - + " tableArchivePath is null."); - } - } - - if (tableDescriptor == null) { - tableDescriptor = new HTableDescriptor(newTableName); - } else { - tableDescriptor.setName(newTableName); - } - - if (!converted) { - // record all region dirs: - // load all files in dir - try { - ArrayList regionPathList = getRegionList(tableName); - - // should only try to create the table with all region informations, so we could pre-split - // the regions in fine grain - checkAndCreateTable(tableBackupPath, tableName, newTableName, regionPathList, - tableDescriptor); - if (tableArchivePath != null) { - // start real restore through bulkload - // if the backup target is on local cluster, special action needed - Path tempTableArchivePath = checkLocalAndBackup(tableArchivePath); - if (tempTableArchivePath.equals(tableArchivePath)) { - if(LOG.isDebugEnabled()) { - LOG.debug("TableArchivePath for bulkload using existPath: " + tableArchivePath); - } - } else { - regionPathList = getRegionList(tempTableArchivePath); // point to the tempDir - if(LOG.isDebugEnabled()) { - LOG.debug("TableArchivePath for bulkload using tempPath: " + tempTableArchivePath); - } - } - - LoadIncrementalHFiles loader = createLoader(tempTableArchivePath, false); - for (Path regionPath : regionPathList) { - String regionName = regionPath.toString(); - if(LOG.isDebugEnabled()) { - LOG.debug("Restoring HFiles from directory " + regionName); - } - String[] args = { regionName, newTableName.getNameAsString()}; - loader.run(args); - } - } - // we do not recovered edits - } catch (Exception e) { - throw new IllegalStateException("Cannot restore hbase table", e); - } - } else { - LOG.debug("convert will be supported in a future jira"); - } - } - - /** - * Gets region list - * @param tableArchivePath table archive path - * @return RegionList region list - * @throws FileNotFoundException exception - * @throws IOException exception - */ - ArrayList getRegionList(Path tableArchivePath) throws FileNotFoundException, - IOException { - ArrayList regionDirList = new ArrayList(); - FileStatus[] children = fs.listStatus(tableArchivePath); - for (FileStatus childStatus : children) { - // here child refer to each region(Name) - Path child = childStatus.getPath(); - regionDirList.add(child); - } - return regionDirList; - } - - /** - * Counts the number of files in all subdirectories of an HBase table, i.e. HFiles. - * @param regionPath Path to an HBase table directory - * @return the number of files all directories - * @throws IOException exception - */ - int getNumberOfFilesInDir(Path regionPath) throws IOException { - int result = 0; - - if (!fs.exists(regionPath) || !fs.getFileStatus(regionPath).isDirectory()) { - throw new IllegalStateException("Cannot restore hbase table because directory '" - + regionPath.toString() + "' is not a directory."); - } - - FileStatus[] tableDirContent = fs.listStatus(regionPath); - for (FileStatus subDirStatus : tableDirContent) { - FileStatus[] colFamilies = fs.listStatus(subDirStatus.getPath()); - for (FileStatus colFamilyStatus : colFamilies) { - FileStatus[] colFamilyContent = fs.listStatus(colFamilyStatus.getPath()); - result += colFamilyContent.length; - } - } - return result; - } - - /** - * Counts the number of files in all subdirectories of an HBase tables, i.e. HFiles. And finds the - * maximum number of files in one HBase table. - * @param tableArchivePath archive path - * @return the maximum number of files found in 1 HBase table - * @throws IOException exception - */ - int getMaxNumberOfFilesInSubDir(Path tableArchivePath) throws IOException { - int result = 1; - ArrayList regionPathList = getRegionList(tableArchivePath); - // tableArchivePath = this.getTableArchivePath(tableName); - - if (regionPathList == null || regionPathList.size() == 0) { - throw new IllegalStateException("Cannot restore hbase table because directory '" - + tableArchivePath + "' is not a directory."); - } - - for (Path regionPath : regionPathList) { - result = Math.max(result, getNumberOfFilesInDir(regionPath)); - } - return result; - } - - /** - * Create a {@link LoadIncrementalHFiles} instance to be used to restore the HFiles of a full - * backup. - * @return the {@link LoadIncrementalHFiles} instance - * @throws IOException exception - */ - private LoadIncrementalHFiles createLoader(Path tableArchivePath, boolean multipleTables) - throws IOException { - // set configuration for restore: - // LoadIncrementalHFile needs more time - // hbase.rpc.timeout 600000 - // calculates - Integer milliSecInMin = 60000; - Integer previousMillis = this.conf.getInt("hbase.rpc.timeout", 0); - Integer numberOfFilesInDir = - multipleTables ? getMaxNumberOfFilesInSubDir(tableArchivePath) : - getNumberOfFilesInDir(tableArchivePath); - Integer calculatedMillis = numberOfFilesInDir * milliSecInMin; // 1 minute per file - Integer resultMillis = Math.max(calculatedMillis, previousMillis); - if (resultMillis > previousMillis) { - LOG.info("Setting configuration for restore with LoadIncrementalHFile: " - + "hbase.rpc.timeout to " + calculatedMillis / milliSecInMin - + " minutes, to handle the number of files in backup " + tableArchivePath); - this.conf.setInt("hbase.rpc.timeout", resultMillis); - } - - // By default, it is 32 and loader will fail if # of files in any region exceed this - // limit. Bad for snapshot restore. - this.conf.setInt(LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, Integer.MAX_VALUE); - LoadIncrementalHFiles loader = null; - try { - loader = new LoadIncrementalHFiles(this.conf); - } catch (Exception e1) { - throw new IOException(e1); - } - return loader; - } - - /** - * Calculate region boundaries and add all the column families to the table descriptor - * @param regionDirList region dir list - * @return a set of keys to store the boundaries - */ - byte[][] generateBoundaryKeys(ArrayList regionDirList) - throws FileNotFoundException, IOException { - TreeMap map = new TreeMap(Bytes.BYTES_COMPARATOR); - // Build a set of keys to store the boundaries - byte[][] keys = null; - // calculate region boundaries and add all the column families to the table descriptor - for (Path regionDir : regionDirList) { - LOG.debug("Parsing region dir: " + regionDir); - Path hfofDir = regionDir; - - if (!fs.exists(hfofDir)) { - LOG.warn("HFileOutputFormat dir " + hfofDir + " not found"); - } - - FileStatus[] familyDirStatuses = fs.listStatus(hfofDir); - if (familyDirStatuses == null) { - throw new IOException("No families found in " + hfofDir); - } - - for (FileStatus stat : familyDirStatuses) { - if (!stat.isDirectory()) { - LOG.warn("Skipping non-directory " + stat.getPath()); - continue; - } - boolean isIgnore = false; - String pathName = stat.getPath().getName(); - for (String ignore : ignoreDirs) { - if (pathName.contains(ignore)) { - LOG.warn("Skipping non-family directory" + pathName); - isIgnore = true; - break; - } - } - if (isIgnore) { - continue; - } - Path familyDir = stat.getPath(); - LOG.debug("Parsing family dir [" + familyDir.toString() + " in region [" + regionDir + "]"); - // Skip _logs, etc - if (familyDir.getName().startsWith("_") || familyDir.getName().startsWith(".")) { - continue; - } - - // start to parse hfile inside one family dir - Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); - for (Path hfile : hfiles) { - if (hfile.getName().startsWith("_") || hfile.getName().startsWith(".") - || StoreFileInfo.isReference(hfile.getName()) - || HFileLink.isHFileLink(hfile.getName())) { - continue; - } - HFile.Reader reader = HFile.createReader(fs, hfile, new CacheConfig(conf), conf); - final byte[] first, last; - try { - reader.loadFileInfo(); - first = reader.getFirstRowKey(); - last = reader.getLastRowKey(); - LOG.debug("Trying to figure out region boundaries hfile=" + hfile + " first=" - + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last)); - - // To eventually infer start key-end key boundaries - Integer value = map.containsKey(first) ? (Integer) map.get(first) : 0; - map.put(first, value + 1); - value = map.containsKey(last) ? (Integer) map.get(last) : 0; - map.put(last, value - 1); - } finally { - reader.close(); - } - } - } - } - keys = LoadIncrementalHFiles.inferBoundaries(map); - return keys; - } - - /** - * Prepare the table for bulkload, most codes copied from - * {@link LoadIncrementalHFiles#createTable(String, String)} - * @param tableBackupPath path - * @param tableName table name - * @param targetTableName target table name - * @param regionDirList region directory list - * @param htd table descriptor - * @throws IOException exception - */ - private void checkAndCreateTable(Path tableBackupPath, TableName tableName, - TableName targetTableName, ArrayList regionDirList, HTableDescriptor htd) - throws IOException { - HBaseAdmin hbadmin = null; - Connection conn = null; - try { - conn = ConnectionFactory.createConnection(conf); - hbadmin = (HBaseAdmin) conn.getAdmin(); - if (hbadmin.tableExists(targetTableName)) { - LOG.info("Using exising target table '" + targetTableName + "'"); - } else { - LOG.info("Creating target table '" + targetTableName + "'"); - - // if no region dir given, create the table and return - if (regionDirList == null || regionDirList.size() == 0) { - - hbadmin.createTable(htd); - return; - } - - byte[][] keys = generateBoundaryKeys(regionDirList); - - // create table using table decriptor and region boundaries - hbadmin.createTable(htd, keys); - } - } catch (Exception e) { - throw new IOException(e); - } finally { - if (hbadmin != null) { - hbadmin.close(); - } - if(conn != null){ - conn.close(); - } - } - } - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java index dceb88f..fd22780 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java @@ -32,10 +32,10 @@ import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupCopyService; import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.impl.BackupCopyService; import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.impl.BackupUtil; +import org.apache.hadoop.hbase.backup.util.BackupServerUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; @@ -205,7 +205,7 @@ public class MapReduceBackupCopyService implements BackupCopyService { long totalSrcLgth = 0; for (Path aSrc : srcs) { - totalSrcLgth += BackupUtil.getFilesLength(aSrc.getFileSystem(getConf()), aSrc); + totalSrcLgth += BackupServerUtil.getFilesLength(aSrc.getFileSystem(getConf()), aSrc); } // submit the copy job diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java index 8cb8125..7c10d3c 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java @@ -27,8 +27,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.impl.BackupUtil; -import org.apache.hadoop.hbase.backup.impl.IncrementalRestoreService; +import org.apache.hadoop.hbase.backup.IncrementalRestoreService; +import org.apache.hadoop.hbase.backup.util.BackupServerUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; @@ -54,7 +54,7 @@ public class MapReduceRestoreService implements IncrementalRestoreService { // log file String logDirs = StringUtils.join(logDirPaths, ","); LOG.info("Restore incremental backup from directory " + logDirs + " from hbase tables " - + BackupUtil.join(tableNames) + " to tables " + BackupUtil.join(newTableNames)); + + BackupServerUtil.join(tableNames) + " to tables " + BackupServerUtil.join(newTableNames)); for (int i = 0; i < tableNames.length; i++) { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java index 5dd67b4..04a91a6 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java @@ -71,7 +71,7 @@ public class BackupLogCleaner extends BaseLogCleanerDelegate { for(FileStatus file: files){ String wal = file.getPath().toString(); - boolean logInSystemTable = table.checkWALFile(wal); + boolean logInSystemTable = table.isWALFileDeletable(wal); if(LOG.isDebugEnabled()) { if(logInSystemTable) { LOG.debug("Found log file in hbase:backup, deleting: " + wal); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java index bb0219b..318c6a6 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java @@ -36,21 +36,21 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupClientUtil; +import org.apache.hadoop.hbase.backup.BackupCopyService; import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupRestoreFactory; +import org.apache.hadoop.hbase.backup.BackupRestoreServerFactory; import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.backup.HBackupFileSystem; +import org.apache.hadoop.hbase.backup.BackupCopyService.Type; import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase; import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.impl.BackupCopyService; import org.apache.hadoop.hbase.backup.impl.BackupException; import org.apache.hadoop.hbase.backup.impl.BackupManager; import org.apache.hadoop.hbase.backup.impl.BackupManifest; import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants; -import org.apache.hadoop.hbase.backup.impl.BackupUtil; -import org.apache.hadoop.hbase.backup.impl.BackupCopyService.Type; import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.backup.util.BackupServerUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; @@ -297,7 +297,7 @@ public class FullTableBackupProcedure // call ExportSnapshot to copy files based on hbase snapshot for backup // ExportSnapshot only support single snapshot export, need loop for multiple tables case - BackupCopyService copyService = BackupRestoreFactory.getBackupCopyService(conf); + BackupCopyService copyService = BackupRestoreServerFactory.getBackupCopyService(conf); // number of snapshots matches number of tables float numOfSnapshots = backupContext.getSnapshotNames().size(); @@ -548,7 +548,7 @@ public class FullTableBackupProcedure // We record ALL old WAL files as registered, because // this is a first full backup in the system and these // files are not needed for next incremental backup - List logFiles = BackupUtil.getWALFilesOlderThan(conf, newTimestamps); + List logFiles = BackupServerUtil.getWALFilesOlderThan(conf, newTimestamps); backupManager.recordWALFiles(logFiles); } } catch (BackupException e) { @@ -581,7 +581,7 @@ public class FullTableBackupProcedure long waitTime = SnapshotDescriptionUtils.getMaxMasterTimeout( env.getMasterConfiguration(), backupSnapshot.getType(), SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME); - BackupUtil.waitForSnapshot(backupSnapshot, waitTime, + BackupServerUtil.waitForSnapshot(backupSnapshot, waitTime, env.getMasterServices().getSnapshotManager(), env.getMasterConfiguration()); // set the snapshot name in BackupStatus of this table, only after snapshot success. backupContext.setSnapshotName(tableName, backupSnapshot.getName()); @@ -616,7 +616,7 @@ public class FullTableBackupProcedure backupManager.readLogTimestampMap(); Long newStartCode = - BackupClientUtil.getMinValue(BackupUtil.getRSLogTimestampMins(newTableSetTimestampMap)); + BackupClientUtil.getMinValue(BackupServerUtil.getRSLogTimestampMins(newTableSetTimestampMap)); backupManager.writeBackupStartCode(newStartCode); // backup complete diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java index 3e4cc1e..62c3dcd 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java @@ -33,17 +33,17 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupClientUtil; +import org.apache.hadoop.hbase.backup.BackupCopyService; import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupRestoreFactory; +import org.apache.hadoop.hbase.backup.BackupRestoreServerFactory; import org.apache.hadoop.hbase.backup.BackupType; +import org.apache.hadoop.hbase.backup.BackupCopyService.Type; import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase; import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; -import org.apache.hadoop.hbase.backup.impl.BackupCopyService; import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.impl.BackupUtil; import org.apache.hadoop.hbase.backup.impl.IncrementalBackupManager; -import org.apache.hadoop.hbase.backup.impl.BackupCopyService.Type; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.backup.util.BackupServerUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface; @@ -125,7 +125,7 @@ public class IncrementalTableBackupProcedure String[] strArr = incrBackupFileList.toArray(new String[incrBackupFileList.size() + 1]); strArr[strArr.length - 1] = backupContext.getHLogTargetDir(); - BackupCopyService copyService = BackupRestoreFactory.getBackupCopyService(conf); + BackupCopyService copyService = BackupRestoreServerFactory.getBackupCopyService(conf); int res = copyService.copy(backupContext, backupManager, conf, BackupCopyService.Type.INCREMENTAL, strArr); @@ -176,7 +176,7 @@ public class IncrementalTableBackupProcedure case INCREMENTAL_COPY: try { // copy out the table and region info files for each table - BackupUtil.copyTableRegionInfo(backupContext, conf); + BackupServerUtil.copyTableRegionInfo(backupContext, conf); incrementalCopy(backupContext); // Save list of WAL files copied backupManager.recordWALFiles(backupContext.getIncrBackupFileList()); @@ -204,7 +204,7 @@ public class IncrementalTableBackupProcedure backupManager.readLogTimestampMap(); Long newStartCode = BackupClientUtil - .getMinValue(BackupUtil.getRSLogTimestampMins(newTableSetTimestampMap)); + .getMinValue(BackupServerUtil.getRSLogTimestampMins(newTableSetTimestampMap)); backupManager.writeBackupStartCode(newStartCode); // backup complete FullTableBackupProcedure.completeBackup(env, backupContext, backupManager, diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java new file mode 100644 index 0000000..6389386 --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java @@ -0,0 +1,477 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.util; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.HBackupFileSystem; +import org.apache.hadoop.hbase.backup.impl.BackupException; +import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.wal.DefaultWALProvider; + +/** + * A collection for methods used by multiple classes to backup HBase tables. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class BackupServerUtil { + protected static final Log LOG = LogFactory.getLog(BackupServerUtil.class); + public static final String LOGNAME_SEPARATOR = "."; + + private BackupServerUtil(){ + throw new AssertionError("Instantiating utility class..."); + } + + public static void waitForSnapshot(SnapshotDescription snapshot, long max, + SnapshotManager snapshotMgr, Configuration conf) throws IOException { + boolean done = false; + long start = EnvironmentEdgeManager.currentTime(); + int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + long maxPauseTime = max / numRetries; + int tries = 0; + LOG.debug("Waiting a max of " + max + " ms for snapshot '" + + ClientSnapshotDescriptionUtils.toString(snapshot) + "'' to complete. (max " + + maxPauseTime + " ms per retry)"); + while (tries == 0 + || ((EnvironmentEdgeManager.currentTime() - start) < max && !done)) { + try { + // sleep a backoff <= pauseTime amount + long pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, + HConstants.DEFAULT_HBASE_CLIENT_PAUSE); + long sleep = HBaseAdmin.getPauseTime(tries++, pause); + sleep = sleep > maxPauseTime ? maxPauseTime : sleep; + LOG.debug("(#" + tries + ") Sleeping: " + sleep + + "ms while waiting for snapshot completion."); + Thread.sleep(sleep); + } catch (InterruptedException e) { + throw (InterruptedIOException)new InterruptedIOException("Interrupted").initCause(e); + } + LOG.debug("Getting current status of snapshot ..."); + done = snapshotMgr.isSnapshotDone(snapshot); + } + if (!done) { + throw new SnapshotCreationException("Snapshot '" + snapshot.getName() + + "' wasn't completed in expectedTime:" + max + " ms", snapshot); + } + } + + /** + * Loop through the RS log timestamp map for the tables, for each RS, find the min timestamp + * value for the RS among the tables. + * @param rsLogTimestampMap timestamp map + * @return the min timestamp of each RS + */ + public static HashMap getRSLogTimestampMins( + HashMap> rsLogTimestampMap) { + + if (rsLogTimestampMap == null || rsLogTimestampMap.isEmpty()) { + return null; + } + + HashMap rsLogTimestampMins = new HashMap(); + HashMap> rsLogTimestampMapByRS = + new HashMap>(); + + for (Entry> tableEntry : rsLogTimestampMap.entrySet()) { + TableName table = tableEntry.getKey(); + HashMap rsLogTimestamp = tableEntry.getValue(); + for (Entry rsEntry : rsLogTimestamp.entrySet()) { + String rs = rsEntry.getKey(); + Long ts = rsEntry.getValue(); + if (!rsLogTimestampMapByRS.containsKey(rs)) { + rsLogTimestampMapByRS.put(rs, new HashMap()); + rsLogTimestampMapByRS.get(rs).put(table, ts); + } else { + rsLogTimestampMapByRS.get(rs).put(table, ts); + } + } + } + + for (String rs : rsLogTimestampMapByRS.keySet()) { + rsLogTimestampMins.put(rs, BackupClientUtil.getMinValue(rsLogTimestampMapByRS.get(rs))); + } + + return rsLogTimestampMins; + } + + /** + * copy out Table RegionInfo into incremental backup image need to consider move this logic into + * HBackupFileSystem + * @param backupContext backup context + * @param conf configuration + * @throws IOException exception + * @throws InterruptedException exception + */ + public static void copyTableRegionInfo(BackupInfo backupContext, Configuration conf) + throws IOException, InterruptedException { + + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = rootDir.getFileSystem(conf); + + // for each table in the table set, copy out the table info and region + // info files in the correct directory structure + try (Connection conn = ConnectionFactory.createConnection(conf); + Admin admin = conn.getAdmin()) { + + for (TableName table : backupContext.getTables()) { + + LOG.debug("Attempting to copy table info for:" + table); + TableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, table); + + // write a copy of descriptor to the target directory + Path target = new Path(backupContext.getBackupStatus(table).getTargetDir()); + FileSystem targetFs = target.getFileSystem(conf); + FSTableDescriptors descriptors = + new FSTableDescriptors(conf, targetFs, FSUtils.getRootDir(conf)); + descriptors.createTableDescriptorForTableDirectory(target, orig, false); + LOG.debug("Finished copying tableinfo."); + List regions = null; + regions = admin.getTableRegions(table); + // For each region, write the region info to disk + LOG.debug("Starting to write region info for table " + table); + for (HRegionInfo regionInfo : regions) { + Path regionDir = + HRegion.getRegionDir(new Path(backupContext.getBackupStatus(table).getTargetDir()), + regionInfo); + regionDir = + new Path(backupContext.getBackupStatus(table).getTargetDir(), regionDir.getName()); + writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo); + } + LOG.debug("Finished writing region info for table " + table); + } + } catch (IOException e) { + throw new BackupException(e); + } + } + + /** + * Write the .regioninfo file on-disk. + */ + public static void writeRegioninfoOnFilesystem(final Configuration conf, final FileSystem fs, + final Path regionInfoDir, HRegionInfo regionInfo) throws IOException { + final byte[] content = regionInfo.toDelimitedByteArray(); + Path regionInfoFile = new Path(regionInfoDir, ".regioninfo"); + // First check to get the permissions + FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY); + // Write the RegionInfo file content + FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null); + try { + out.write(content); + } finally { + out.close(); + } + } + + /** + * TODO: return hostname:port + * @param p + * @return host name: port + * @throws IOException + */ + public static String parseHostNameFromLogFile(Path p) throws IOException { + if (isArchivedLogFile(p)) { + return BackupClientUtil.parseHostFromOldLog(p); + } else { + ServerName sname = DefaultWALProvider.getServerNameFromWALDirectoryName(p); + return sname.getHostname() + ":" + sname.getPort(); + } + } + + private static boolean isArchivedLogFile(Path p) { + String oldLog = Path.SEPARATOR + HConstants.HREGION_OLDLOGDIR_NAME + Path.SEPARATOR; + return p.toString().contains(oldLog); + } + + /** + * Returns WAL file name + * @param walFileName WAL file name + * @return WAL file name + * @throws IOException exception + * @throws IllegalArgumentException exception + */ + public static String getUniqueWALFileNamePart(String walFileName) throws IOException { + return getUniqueWALFileNamePart(new Path(walFileName)); + } + + /** + * Returns WAL file name + * @param p - WAL file path + * @return WAL file name + * @throws IOException exception + */ + public static String getUniqueWALFileNamePart(Path p) throws IOException { + return p.getName(); + } + + /** + * Get the total length of files under the given directory recursively. + * @param fs The hadoop file system + * @param dir The target directory + * @return the total length of files + * @throws IOException exception + */ + public static long getFilesLength(FileSystem fs, Path dir) throws IOException { + long totalLength = 0; + FileStatus[] files = FSUtils.listStatus(fs, dir); + if (files != null) { + for (FileStatus fileStatus : files) { + if (fileStatus.isDirectory()) { + totalLength += getFilesLength(fs, fileStatus.getPath()); + } else { + totalLength += fileStatus.getLen(); + } + } + } + return totalLength; + } + + + + /** + * Sort history list by start time in descending order. + * @param historyList history list + * @return sorted list of BackupCompleteData + */ + public static ArrayList sortHistoryListDesc( + ArrayList historyList) { + ArrayList list = new ArrayList(); + TreeMap map = new TreeMap(); + for (BackupInfo h : historyList) { + map.put(Long.toString(h.getStartTs()), h); + } + Iterator i = map.descendingKeySet().iterator(); + while (i.hasNext()) { + list.add(map.get(i.next())); + } + return list; + } + + /** + * Get list of all WAL files (WALs and archive) + * @param c - configuration + * @return list of WAL files + * @throws IOException exception + */ + public static List getListOfWALFiles(Configuration c) throws IOException { + Path rootDir = FSUtils.getRootDir(c); + Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); + Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + List logFiles = new ArrayList(); + + FileSystem fs = FileSystem.get(c); + logFiles = BackupClientUtil.getFiles(fs, logDir, logFiles, null); + logFiles = BackupClientUtil.getFiles(fs, oldLogDir, logFiles, null); + return logFiles; + } + + /** + * Get list of all WAL files (WALs and archive) + * @param c - configuration + * @return list of WAL files + * @throws IOException exception + */ + public static List getListOfWALFiles(Configuration c, PathFilter filter) + throws IOException { + Path rootDir = FSUtils.getRootDir(c); + Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); + Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + List logFiles = new ArrayList(); + + FileSystem fs = FileSystem.get(c); + logFiles = BackupClientUtil.getFiles(fs, logDir, logFiles, filter); + logFiles = BackupClientUtil.getFiles(fs, oldLogDir, logFiles, filter); + return logFiles; + } + + /** + * Get list of all old WAL files (WALs and archive) + * @param c - configuration + * @param hostTimestampMap - host timestamp map + * @return list of WAL files + * @throws IOException exception + */ + public static List getWALFilesOlderThan(final Configuration c, + final HashMap hostTimestampMap) throws IOException { + Path rootDir = FSUtils.getRootDir(c); + Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME); + Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + List logFiles = new ArrayList(); + + PathFilter filter = new PathFilter() { + + @Override + public boolean accept(Path p) { + try { + if (DefaultWALProvider.isMetaFile(p)) { + return false; + } + String host = parseHostNameFromLogFile(p); + Long oldTimestamp = hostTimestampMap.get(host); + Long currentLogTS = BackupClientUtil.getCreationTime(p); + return currentLogTS <= oldTimestamp; + } catch (IOException e) { + LOG.error(e); + return false; + } + } + }; + FileSystem fs = FileSystem.get(c); + logFiles = BackupClientUtil.getFiles(fs, logDir, logFiles, filter); + logFiles = BackupClientUtil.getFiles(fs, oldLogDir, logFiles, filter); + return logFiles; + } + + public static String join(TableName[] names) { + StringBuilder sb = new StringBuilder(); + String sep = BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND; + for (TableName s : names) { + sb.append(sep).append(s.getNameAsString()); + } + return sb.toString(); + } + + public static TableName[] parseTableNames(String tables) { + if (tables == null) { + return null; + } + String[] tableArray = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); + + TableName[] ret = new TableName[tableArray.length]; + for (int i = 0; i < tableArray.length; i++) { + ret[i] = TableName.valueOf(tableArray[i]); + } + return ret; + } + + public static void cleanupBackupData(BackupInfo context, Configuration conf) + throws IOException + { + cleanupHLogDir(context, conf); + cleanupTargetDir(context, conf); + } + + /** + * Clean up directories which are generated when DistCp copying hlogs. + * @throws IOException + */ + private static void cleanupHLogDir(BackupInfo backupContext, Configuration conf) + throws IOException { + + String logDir = backupContext.getHLogTargetDir(); + if (logDir == null) { + LOG.warn("No log directory specified for " + backupContext.getBackupId()); + return; + } + + Path rootPath = new Path(logDir).getParent(); + FileSystem fs = FileSystem.get(rootPath.toUri(), conf); + FileStatus[] files = FSUtils.listStatus(fs, rootPath); + if (files == null) { + return; + } + for (FileStatus file : files) { + LOG.debug("Delete log files: " + file.getPath().getName()); + if(!FSUtils.delete(fs, file.getPath(), true)) { + LOG.warn("Could not delete files in "+ file.getPath()); + }; + } + } + + /** + * Clean up the data at target directory + */ + private static void cleanupTargetDir(BackupInfo backupContext, Configuration conf) { + try { + // clean up the data at target directory + LOG.debug("Trying to cleanup up target dir : " + backupContext.getBackupId()); + String targetDir = backupContext.getTargetRootDir(); + if (targetDir == null) { + LOG.warn("No target directory specified for " + backupContext.getBackupId()); + return; + } + + FileSystem outputFs = + FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf); + + for (TableName table : backupContext.getTables()) { + Path targetDirPath = + new Path(HBackupFileSystem.getTableBackupDir(backupContext.getTargetRootDir(), + backupContext.getBackupId(), table)); + if (outputFs.delete(targetDirPath, true)) { + LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); + } else { + LOG.info("No data has been found in " + targetDirPath.toString() + "."); + } + + Path tableDir = targetDirPath.getParent(); + FileStatus[] backups = FSUtils.listStatus(outputFs, tableDir); + if (backups == null || backups.length == 0) { + if(outputFs.delete(tableDir, true)){ + LOG.debug(tableDir.toString() + " is empty, remove it."); + } else { + LOG.warn("Could not delete "+ tableDir); + } + } + } + + } catch (IOException e1) { + LOG.error("Cleaning up backup data of " + backupContext.getBackupId() + " at " + + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + "."); + } + } + + +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java deleted file mode 100644 index 76402c7..0000000 --- hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup.util; -import java.util.List; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -/** - * Backup set is a named group of HBase tables, - * which are managed together by Backup/Restore - * framework. Instead of using list of tables in backup or restore - * operation, one can use set's name instead. - */ -@InterfaceAudience.Public -@InterfaceStability.Evolving -public class BackupSet { - private final String name; - private final List tables; - - public BackupSet(String name, List tables) { - this.name = name; - this.tables = tables; - } - - public String getName() { - return name; - } - - public List getTables() { - return tables; - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(name).append("={"); - for (int i = 0; i < tables.size(); i++) { - sb.append(tables.get(i)); - if (i < tables.size() - 1) { - sb.append(","); - } - } - sb.append("}"); - return sb.toString(); - } - -} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java new file mode 100644 index 0000000..37631bb --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreServerUtil.java @@ -0,0 +1,609 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.util; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupRestoreServerFactory; +import org.apache.hadoop.hbase.backup.HBackupFileSystem; +import org.apache.hadoop.hbase.backup.IncrementalRestoreService; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.HStore; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; +import org.apache.hadoop.hbase.snapshot.SnapshotManifest; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * A collection for methods used by multiple classes to restore HBase tables. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class RestoreServerUtil { + + public static final Log LOG = LogFactory.getLog(RestoreServerUtil.class); + + private final String[] ignoreDirs = { "recovered.edits" }; + + protected Configuration conf = null; + + protected Path backupRootPath; + + protected String backupId; + + protected FileSystem fs; + private final String RESTORE_TMP_PATH = "/tmp"; + private final Path restoreTmpPath; + + // store table name and snapshot dir mapping + private final HashMap snapshotMap = new HashMap<>(); + + public RestoreServerUtil(Configuration conf, final Path backupRootPath, final String backupId) + throws IOException { + this.conf = conf; + this.backupRootPath = backupRootPath; + this.backupId = backupId; + this.fs = backupRootPath.getFileSystem(conf); + this.restoreTmpPath = new Path(conf.get("hbase.fs.tmp.dir") != null? + conf.get("hbase.fs.tmp.dir"): RESTORE_TMP_PATH, + "restore"); + } + + /** + * return value represent path for: + * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/archive/data/default/t1_dn" + * @param tabelName table name + * @return path to table archive + * @throws IOException exception + */ + Path getTableArchivePath(TableName tableName) + throws IOException { + Path baseDir = new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, + backupId), HConstants.HFILE_ARCHIVE_DIRECTORY); + Path dataDir = new Path(baseDir, HConstants.BASE_NAMESPACE_DIR); + Path archivePath = new Path(dataDir, tableName.getNamespaceAsString()); + Path tableArchivePath = + new Path(archivePath, tableName.getQualifierAsString()); + if (!fs.exists(tableArchivePath) || !fs.getFileStatus(tableArchivePath).isDirectory()) { + LOG.debug("Folder tableArchivePath: " + tableArchivePath.toString() + " does not exists"); + tableArchivePath = null; // empty table has no archive + } + return tableArchivePath; + } + + /** + * Gets region list + * @param tableName table name + * @return RegionList region list + * @throws FileNotFoundException exception + * @throws IOException exception + */ + ArrayList getRegionList(TableName tableName) + throws FileNotFoundException, IOException { + Path tableArchivePath = this.getTableArchivePath(tableName); + ArrayList regionDirList = new ArrayList(); + FileStatus[] children = fs.listStatus(tableArchivePath); + for (FileStatus childStatus : children) { + // here child refer to each region(Name) + Path child = childStatus.getPath(); + regionDirList.add(child); + } + return regionDirList; + } + + /** + * During incremental backup operation. Call WalPlayer to replay WAL in backup image Currently + * tableNames and newTablesNames only contain single table, will be expanded to multiple tables in + * the future + * @param logDir : incremental backup folders, which contains WAL + * @param tableNames : source tableNames(table names were backuped) + * @param newTableNames : target tableNames(table names to be restored to) + * @throws IOException exception + */ + public void incrementalRestoreTable(Path[] logDirs, + TableName[] tableNames, TableName[] newTableNames) throws IOException { + + if (tableNames.length != newTableNames.length) { + throw new IOException("Number of source tables and target tables does not match!"); + } + + // for incremental backup image, expect the table already created either by user or previous + // full backup. Here, check that all new tables exists + try (Connection conn = ConnectionFactory.createConnection(conf); + Admin admin = conn.getAdmin()) { + for (TableName tableName : newTableNames) { + if (!admin.tableExists(tableName)) { + admin.close(); + throw new IOException("HBase table " + tableName + + " does not exist. Create the table first, e.g. by restoring a full backup."); + } + } + IncrementalRestoreService restoreService = + BackupRestoreServerFactory.getIncrementalRestoreService(conf); + + restoreService.run(logDirs, tableNames, newTableNames); + } + } + + public void fullRestoreTable(Path tableBackupPath, TableName tableName, TableName newTableName, + boolean converted) throws IOException { + restoreTableAndCreate(tableName, newTableName, tableBackupPath, converted); + } + + /** + * return value represent path for: + * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/.hbase-snapshot" + * @param backupRootPath backup root path + * @param tableName table name + * @param backupId backup Id + * @return path for snapshot + */ + static Path getTableSnapshotPath(Path backupRootPath, TableName tableName, + String backupId) { + return new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, backupId), + HConstants.SNAPSHOT_DIR_NAME); + } + + /** + * return value represent path for: + * "..../default/t1_dn/backup_1396650096738/.hbase-snapshot/snapshot_1396650097621_default_t1_dn" + * this path contains .snapshotinfo, .tabledesc (0.96 and 0.98) this path contains .snapshotinfo, + * .data.manifest (trunk) + * @param tableName table name + * @return path to table info + * @throws FileNotFoundException exception + * @throws IOException exception + */ + Path getTableInfoPath(TableName tableName) + throws FileNotFoundException, IOException { + Path tableSnapShotPath = getTableSnapshotPath(backupRootPath, tableName, backupId); + Path tableInfoPath = null; + + // can't build the path directly as the timestamp values are different + FileStatus[] snapshots = fs.listStatus(tableSnapShotPath); + for (FileStatus snapshot : snapshots) { + tableInfoPath = snapshot.getPath(); + // SnapshotManifest.DATA_MANIFEST_NAME = "data.manifest"; + if (tableInfoPath.getName().endsWith("data.manifest")) { + break; + } + } + return tableInfoPath; + } + + /** + * @param tableName is the table backed up + * @return {@link HTableDescriptor} saved in backup image of the table + */ + HTableDescriptor getTableDesc(TableName tableName) + throws FileNotFoundException, IOException { + Path tableInfoPath = this.getTableInfoPath(tableName); + SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath); + SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc); + HTableDescriptor tableDescriptor = manifest.getTableDescriptor(); + if (!tableDescriptor.getTableName().equals(tableName)) { + LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: " + + tableInfoPath.toString()); + LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getNameAsString()); + throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName + + " under tableInfoPath: " + tableInfoPath.toString()); + } + return tableDescriptor; + } + + /** + * Duplicate the backup image if it's on local cluster + * @see HStore#bulkLoadHFile(String, long) + * @see HRegionFileSystem#bulkLoadStoreFile(String familyName, Path srcPath, long seqNum) + * @param tableArchivePath archive path + * @return the new tableArchivePath + * @throws IOException exception + */ + Path checkLocalAndBackup(Path tableArchivePath) throws IOException { + // Move the file if it's on local cluster + boolean isCopyNeeded = false; + + FileSystem srcFs = tableArchivePath.getFileSystem(conf); + FileSystem desFs = FileSystem.get(conf); + if (tableArchivePath.getName().startsWith("/")) { + isCopyNeeded = true; + } else { + // This should match what is done in @see HRegionFileSystem#bulkLoadStoreFile(String, Path, + // long) + if (srcFs.getUri().equals(desFs.getUri())) { + LOG.debug("cluster hold the backup image: " + srcFs.getUri() + "; local cluster node: " + + desFs.getUri()); + isCopyNeeded = true; + } + } + if (isCopyNeeded) { + LOG.debug("File " + tableArchivePath + " on local cluster, back it up before restore"); + if (desFs.exists(restoreTmpPath)) { + try { + desFs.delete(restoreTmpPath, true); + } catch (IOException e) { + LOG.debug("Failed to delete path: " + restoreTmpPath + + ", need to check whether restore target DFS cluster is healthy"); + } + } + FileUtil.copy(srcFs, tableArchivePath, desFs, restoreTmpPath, false, conf); + LOG.debug("Copied to temporary path on local cluster: " + restoreTmpPath); + tableArchivePath = restoreTmpPath; + } + return tableArchivePath; + } + + private void restoreTableAndCreate(TableName tableName, TableName newTableName, + Path tableBackupPath, boolean converted) throws IOException { + if (newTableName == null || newTableName.equals("")) { + newTableName = tableName; + } + + FileSystem fileSys = tableBackupPath.getFileSystem(this.conf); + + // get table descriptor first + HTableDescriptor tableDescriptor = null; + + Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId); + + if (fileSys.exists(tableSnapshotPath)) { + // snapshot path exist means the backup path is in HDFS + // check whether snapshot dir already recorded for target table + if (snapshotMap.get(tableName) != null) { + SnapshotDescription desc = + SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath); + SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc); + tableDescriptor = manifest.getTableDescriptor(); + } else { + tableDescriptor = getTableDesc(tableName); + snapshotMap.put(tableName, getTableInfoPath(tableName)); + } + if (tableDescriptor == null) { + LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost"); + } + } else if (converted) { + // first check if this is a converted backup image + LOG.error("convert will be supported in a future jira"); + } + + Path tableArchivePath = getTableArchivePath(tableName); + if (tableArchivePath == null) { + if (tableDescriptor != null) { + // find table descriptor but no archive dir means the table is empty, create table and exit + if(LOG.isDebugEnabled()) { + LOG.debug("find table descriptor but no archive dir for table " + tableName + + ", will only create table"); + } + tableDescriptor.setName(newTableName); + checkAndCreateTable(tableBackupPath, tableName, newTableName, null, tableDescriptor); + return; + } else { + throw new IllegalStateException("Cannot restore hbase table because directory '" + + " tableArchivePath is null."); + } + } + + if (tableDescriptor == null) { + tableDescriptor = new HTableDescriptor(newTableName); + } else { + tableDescriptor.setName(newTableName); + } + + if (!converted) { + // record all region dirs: + // load all files in dir + try { + ArrayList regionPathList = getRegionList(tableName); + + // should only try to create the table with all region informations, so we could pre-split + // the regions in fine grain + checkAndCreateTable(tableBackupPath, tableName, newTableName, regionPathList, + tableDescriptor); + if (tableArchivePath != null) { + // start real restore through bulkload + // if the backup target is on local cluster, special action needed + Path tempTableArchivePath = checkLocalAndBackup(tableArchivePath); + if (tempTableArchivePath.equals(tableArchivePath)) { + if(LOG.isDebugEnabled()) { + LOG.debug("TableArchivePath for bulkload using existPath: " + tableArchivePath); + } + } else { + regionPathList = getRegionList(tempTableArchivePath); // point to the tempDir + if(LOG.isDebugEnabled()) { + LOG.debug("TableArchivePath for bulkload using tempPath: " + tempTableArchivePath); + } + } + + LoadIncrementalHFiles loader = createLoader(tempTableArchivePath, false); + for (Path regionPath : regionPathList) { + String regionName = regionPath.toString(); + if(LOG.isDebugEnabled()) { + LOG.debug("Restoring HFiles from directory " + regionName); + } + String[] args = { regionName, newTableName.getNameAsString()}; + loader.run(args); + } + } + // we do not recovered edits + } catch (Exception e) { + throw new IllegalStateException("Cannot restore hbase table", e); + } + } else { + LOG.debug("convert will be supported in a future jira"); + } + } + + /** + * Gets region list + * @param tableArchivePath table archive path + * @return RegionList region list + * @throws FileNotFoundException exception + * @throws IOException exception + */ + ArrayList getRegionList(Path tableArchivePath) throws FileNotFoundException, + IOException { + ArrayList regionDirList = new ArrayList(); + FileStatus[] children = fs.listStatus(tableArchivePath); + for (FileStatus childStatus : children) { + // here child refer to each region(Name) + Path child = childStatus.getPath(); + regionDirList.add(child); + } + return regionDirList; + } + + /** + * Counts the number of files in all subdirectories of an HBase table, i.e. HFiles. + * @param regionPath Path to an HBase table directory + * @return the number of files all directories + * @throws IOException exception + */ + int getNumberOfFilesInDir(Path regionPath) throws IOException { + int result = 0; + + if (!fs.exists(regionPath) || !fs.getFileStatus(regionPath).isDirectory()) { + throw new IllegalStateException("Cannot restore hbase table because directory '" + + regionPath.toString() + "' is not a directory."); + } + + FileStatus[] tableDirContent = fs.listStatus(regionPath); + for (FileStatus subDirStatus : tableDirContent) { + FileStatus[] colFamilies = fs.listStatus(subDirStatus.getPath()); + for (FileStatus colFamilyStatus : colFamilies) { + FileStatus[] colFamilyContent = fs.listStatus(colFamilyStatus.getPath()); + result += colFamilyContent.length; + } + } + return result; + } + + /** + * Counts the number of files in all subdirectories of an HBase tables, i.e. HFiles. And finds the + * maximum number of files in one HBase table. + * @param tableArchivePath archive path + * @return the maximum number of files found in 1 HBase table + * @throws IOException exception + */ + int getMaxNumberOfFilesInSubDir(Path tableArchivePath) throws IOException { + int result = 1; + ArrayList regionPathList = getRegionList(tableArchivePath); + // tableArchivePath = this.getTableArchivePath(tableName); + + if (regionPathList == null || regionPathList.size() == 0) { + throw new IllegalStateException("Cannot restore hbase table because directory '" + + tableArchivePath + "' is not a directory."); + } + + for (Path regionPath : regionPathList) { + result = Math.max(result, getNumberOfFilesInDir(regionPath)); + } + return result; + } + + /** + * Create a {@link LoadIncrementalHFiles} instance to be used to restore the HFiles of a full + * backup. + * @return the {@link LoadIncrementalHFiles} instance + * @throws IOException exception + */ + private LoadIncrementalHFiles createLoader(Path tableArchivePath, boolean multipleTables) + throws IOException { + // set configuration for restore: + // LoadIncrementalHFile needs more time + // hbase.rpc.timeout 600000 + // calculates + Integer milliSecInMin = 60000; + Integer previousMillis = this.conf.getInt("hbase.rpc.timeout", 0); + Integer numberOfFilesInDir = + multipleTables ? getMaxNumberOfFilesInSubDir(tableArchivePath) : + getNumberOfFilesInDir(tableArchivePath); + Integer calculatedMillis = numberOfFilesInDir * milliSecInMin; // 1 minute per file + Integer resultMillis = Math.max(calculatedMillis, previousMillis); + if (resultMillis > previousMillis) { + LOG.info("Setting configuration for restore with LoadIncrementalHFile: " + + "hbase.rpc.timeout to " + calculatedMillis / milliSecInMin + + " minutes, to handle the number of files in backup " + tableArchivePath); + this.conf.setInt("hbase.rpc.timeout", resultMillis); + } + + // By default, it is 32 and loader will fail if # of files in any region exceed this + // limit. Bad for snapshot restore. + this.conf.setInt(LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, Integer.MAX_VALUE); + LoadIncrementalHFiles loader = null; + try { + loader = new LoadIncrementalHFiles(this.conf); + } catch (Exception e1) { + throw new IOException(e1); + } + return loader; + } + + /** + * Calculate region boundaries and add all the column families to the table descriptor + * @param regionDirList region dir list + * @return a set of keys to store the boundaries + */ + byte[][] generateBoundaryKeys(ArrayList regionDirList) + throws FileNotFoundException, IOException { + TreeMap map = new TreeMap(Bytes.BYTES_COMPARATOR); + // Build a set of keys to store the boundaries + byte[][] keys = null; + // calculate region boundaries and add all the column families to the table descriptor + for (Path regionDir : regionDirList) { + LOG.debug("Parsing region dir: " + regionDir); + Path hfofDir = regionDir; + + if (!fs.exists(hfofDir)) { + LOG.warn("HFileOutputFormat dir " + hfofDir + " not found"); + } + + FileStatus[] familyDirStatuses = fs.listStatus(hfofDir); + if (familyDirStatuses == null) { + throw new IOException("No families found in " + hfofDir); + } + + for (FileStatus stat : familyDirStatuses) { + if (!stat.isDirectory()) { + LOG.warn("Skipping non-directory " + stat.getPath()); + continue; + } + boolean isIgnore = false; + String pathName = stat.getPath().getName(); + for (String ignore : ignoreDirs) { + if (pathName.contains(ignore)) { + LOG.warn("Skipping non-family directory" + pathName); + isIgnore = true; + break; + } + } + if (isIgnore) { + continue; + } + Path familyDir = stat.getPath(); + LOG.debug("Parsing family dir [" + familyDir.toString() + " in region [" + regionDir + "]"); + // Skip _logs, etc + if (familyDir.getName().startsWith("_") || familyDir.getName().startsWith(".")) { + continue; + } + + // start to parse hfile inside one family dir + Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); + for (Path hfile : hfiles) { + if (hfile.getName().startsWith("_") || hfile.getName().startsWith(".") + || StoreFileInfo.isReference(hfile.getName()) + || HFileLink.isHFileLink(hfile.getName())) { + continue; + } + HFile.Reader reader = HFile.createReader(fs, hfile, new CacheConfig(conf), conf); + final byte[] first, last; + try { + reader.loadFileInfo(); + first = reader.getFirstRowKey(); + last = reader.getLastRowKey(); + LOG.debug("Trying to figure out region boundaries hfile=" + hfile + " first=" + + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last)); + + // To eventually infer start key-end key boundaries + Integer value = map.containsKey(first) ? (Integer) map.get(first) : 0; + map.put(first, value + 1); + value = map.containsKey(last) ? (Integer) map.get(last) : 0; + map.put(last, value - 1); + } finally { + reader.close(); + } + } + } + } + keys = LoadIncrementalHFiles.inferBoundaries(map); + return keys; + } + + /** + * Prepare the table for bulkload, most codes copied from + * {@link LoadIncrementalHFiles#createTable(String, String)} + * @param tableBackupPath path + * @param tableName table name + * @param targetTableName target table name + * @param regionDirList region directory list + * @param htd table descriptor + * @throws IOException exception + */ + private void checkAndCreateTable(Path tableBackupPath, TableName tableName, + TableName targetTableName, ArrayList regionDirList, HTableDescriptor htd) + throws IOException { + HBaseAdmin hbadmin = null; + Connection conn = null; + try { + conn = ConnectionFactory.createConnection(conf); + hbadmin = (HBaseAdmin) conn.getAdmin(); + if (hbadmin.tableExists(targetTableName)) { + LOG.info("Using exising target table '" + targetTableName + "'"); + } else { + LOG.info("Creating target table '" + targetTableName + "'"); + + // if no region dir given, create the table and return + if (regionDirList == null || regionDirList.size() == 0) { + + hbadmin.createTable(htd); + return; + } + + byte[][] keys = generateBoundaryKeys(regionDirList); + + // create table using table decriptor and region boundaries + hbadmin.createTable(htd, keys); + } + } catch (Exception e) { + throw new IOException(e); + } finally { + if (hbadmin != null) { + hbadmin.close(); + } + if(conn != null){ + conn.close(); + } + } + } + +} diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupAdmin.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupAdmin.java new file mode 100644 index 0000000..3630d87 --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupAdmin.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.BackupAdmin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.hamcrest.CoreMatchers; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +@Category(LargeTests.class) +public class TestBackupAdmin extends TestBackupBase { + private static final Log LOG = LogFactory.getLog(TestBackupAdmin.class); + //implement all test cases in 1 test since incremental backup/restore has dependencies + @Test + public void TestIncBackupRestoreWithAdminAPI() throws Exception { + // #1 - create full backup for all tables + LOG.info("create full backup image for all tables"); + + List tables = Lists.newArrayList(table1, table2, table3, table4); + HBaseAdmin admin = null; + BackupAdmin backupAdmin = null; + Connection conn = ConnectionFactory.createConnection(conf1); + admin = (HBaseAdmin) conn.getAdmin(); + backupAdmin = admin.getBackupAdmin(); + BackupRequest request = new BackupRequest(); + request.setBackupType(BackupType.FULL).setTableList(tables).setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdFull = backupAdmin.backupTables(request); + + assertTrue(checkSucceeded(backupIdFull)); + + // #2 - insert some data to table + HTable t1 = (HTable) conn.getTable(table1); + Put p1; + for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { + p1 = new Put(Bytes.toBytes("row-t1" + i)); + p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t1.put(p1); + } + + Assert.assertThat(TEST_UTIL.countRows(t1), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2)); + t1.close(); + + HTable t2 = (HTable) conn.getTable(table2); + Put p2; + for (int i = 0; i < 5; i++) { + p2 = new Put(Bytes.toBytes("row-t2" + i)); + p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t2.put(p2); + } + + Assert.assertThat(TEST_UTIL.countRows(t2), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5)); + t2.close(); + + // #3 - incremental backup for multiple tables + tables = Lists.newArrayList(table1, table2, table3); + request = new BackupRequest(); + request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) + .setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdIncMultiple = backupAdmin.backupTables(request); + assertTrue(checkSucceeded(backupIdIncMultiple)); + + // #4 - restore full backup for all tables, without overwrite + TableName[] tablesRestoreFull = + new TableName[] { table1, table2, table3, table4 }; + + TableName[] tablesMapFull = + new TableName[] { table1_restore, table2_restore, table3_restore, table4_restore }; + + RestoreRequest restoreRequest = new RestoreRequest(); + restoreRequest.setBackupRootDir(BACKUP_ROOT_DIR).setBackupId(backupIdFull). + setCheck(false).setAutorestore(false).setOverwrite(false). + setFromTables(tablesRestoreFull).setToTables(tablesMapFull); + + backupAdmin.restore(restoreRequest); + + // #5.1 - check tables for full restore + + assertTrue(admin.tableExists(table1_restore)); + assertTrue(admin.tableExists(table2_restore)); + assertTrue(admin.tableExists(table3_restore)); + assertTrue(admin.tableExists(table4_restore)); + + + // #5.2 - checking row count of tables for full restore + HTable hTable = (HTable) conn.getTable(table1_restore); + Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH)); + hTable.close(); + + hTable = (HTable) conn.getTable(table2_restore); + Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH)); + hTable.close(); + + hTable = (HTable) conn.getTable(table3_restore); + Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0)); + hTable.close(); + + hTable = (HTable) conn.getTable(table4_restore); + Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0)); + hTable.close(); + + // #6 - restore incremental backup for multiple tables, with overwrite + TableName[] tablesRestoreIncMultiple = + new TableName[] { table1, table2, table3 }; + TableName[] tablesMapIncMultiple = + new TableName[] { table1_restore, table2_restore, table3_restore }; + + restoreRequest = new RestoreRequest(); + restoreRequest.setBackupRootDir(BACKUP_ROOT_DIR).setBackupId(backupIdIncMultiple). + setCheck(false).setAutorestore(false).setOverwrite(true). + setFromTables(tablesRestoreIncMultiple).setToTables(tablesMapIncMultiple); + + backupAdmin.restore(restoreRequest); + + hTable = (HTable) conn.getTable(table1_restore); + Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2)); + hTable.close(); + + hTable = (HTable) conn.getTable(table2_restore); + Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5)); + hTable.close(); + + hTable = (HTable) conn.getTable(table3_restore); + Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0)); + hTable.close(); + + // #7 - incremental backup for single, empty table + + tables = toList(table4.getNameAsString()); + request = new BackupRequest(); + request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) + .setTargetRootDir(BACKUP_ROOT_DIR); + String backupIdIncEmpty = admin.getBackupAdmin().backupTables(request); + + + // #8 - restore incremental backup for single empty table, with overwrite + TableName[] tablesRestoreIncEmpty = new TableName[] { table4 }; + TableName[] tablesMapIncEmpty = new TableName[] { table4_restore }; + + restoreRequest = new RestoreRequest(); + restoreRequest.setBackupRootDir(BACKUP_ROOT_DIR).setBackupId(backupIdIncEmpty). + setCheck(false).setAutorestore(false).setOverwrite(true). + setFromTables(tablesRestoreIncEmpty).setToTables(tablesMapIncEmpty); + + backupAdmin.restore(restoreRequest); + + hTable = (HTable) conn.getTable(table4_restore); + Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0)); + hTable.close(); + admin.close(); + conn.close(); + } + +} diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 8358e47..4d06650 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.BackupAdmin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.HBaseAdmin; @@ -144,14 +145,19 @@ public class TestBackupBase { throws IOException { Connection conn = null; HBaseAdmin admin = null; + BackupAdmin badmin = null; String backupId; try { conn = ConnectionFactory.createConnection(conf1); admin = (HBaseAdmin) conn.getAdmin(); BackupRequest request = new BackupRequest(); request.setBackupType(type).setTableList(tables).setTargetRootDir(path); - backupId = admin.backupTables(request); + badmin = admin.getBackupAdmin(); + backupId = badmin.backupTables(request); } finally { + if(badmin != null){ + badmin.close(); + } if (admin != null) { admin.close(); } @@ -229,15 +235,33 @@ public class TestBackupBase { } } - protected BackupClient getBackupClient(){ - return BackupRestoreFactory.getBackupClient(conf1); - } +// protected BackupClient getBackupClient(){ +// return BackupRestoreClientFactory.getBackupClient(conf1); +// } - protected RestoreClient getRestoreClient() - { - return BackupRestoreFactory.getRestoreClient(conf1); - } +// protected RestoreClient getRestoreClient() +// { +// return BackupRestoreClientFactory.getRestoreClient(conf1); +// } + protected BackupAdmin getBackupAdmin() throws IOException { + return TEST_UTIL.getAdmin().getBackupAdmin(); + } + + /** + * Get restore request. + * + */ + public RestoreRequest createRestoreRequest( + String backupRootDir, + String backupId, boolean check, boolean autoRestore, TableName[] fromTables, + TableName[] toTables, boolean isOverwrite) { + RestoreRequest request = new RestoreRequest(); + request.setBackupRootDir(backupRootDir).setBackupId(backupId).setCheck(check).setAutorestore(autoRestore). + setFromTables(fromTables).setToTables(toTables).setOverwrite(isOverwrite); + return request; +} + /** * Helper method */ diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java index eeb89b5..192d8e8 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java @@ -51,16 +51,10 @@ public class TestBackupDelete extends TestBackupBase { String backupId = fullTableBackup(tableList); assertTrue(checkSucceeded(backupId)); LOG.info("backup complete"); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - System.setOut(new PrintStream(baos)); - String[] backupIds = new String[] { backupId }; - getBackupClient().deleteBackups(backupIds); - + int deleted = getBackupAdmin().deleteBackups(backupIds); + assertTrue(1 == deleted); LOG.info("delete_backup"); - String output = baos.toString(); - LOG.info(baos.toString()); - assertTrue(output.indexOf("Delete backup for backupID=" + backupId + " completed.") >= 0); } /** @@ -90,7 +84,7 @@ public class TestBackupDelete extends TestBackupBase { LOG.info("delete_backup"); String output = baos.toString(); LOG.info(baos.toString()); - assertTrue(output.indexOf("Delete backup for backupID=" + backupId + " completed.") >= 0); + assertTrue(output.indexOf("Deleted 1 backups") >= 0); } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java index 4f7cb11..4a35a06 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDescribe.java @@ -58,7 +58,7 @@ public class TestBackupDescribe extends TestBackupBase { assertTrue(checkSucceeded(backupId)); - BackupInfo info = getBackupClient().getBackupInfo(backupId); + BackupInfo info = getBackupAdmin().getBackupInfo(backupId); assertTrue(info.getState() == BackupState.COMPLETE); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java index 716a22a..a7d2750 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java @@ -54,7 +54,7 @@ public class TestBackupShowHistory extends TestBackupBase { assertTrue(checkSucceeded(backupId)); LOG.info("backup complete"); - List history = getBackupClient().getHistory(10); + List history = getBackupAdmin().getHistory(10); assertTrue(history.size() > 0); boolean success = false; for(BackupInfo info: history){ diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java index ce04b0b..be4019b 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupStatusProgress.java @@ -55,9 +55,9 @@ public class TestBackupStatusProgress extends TestBackupBase { assertTrue(checkSucceeded(backupId)); - BackupInfo info = getBackupClient().getBackupInfo(backupId); + BackupInfo info = getBackupAdmin().getBackupInfo(backupId); assertTrue(info.getState() == BackupState.COMPLETE); - int p = getBackupClient().getProgress(backupId); + int p = getBackupAdmin().getProgress(backupId); LOG.debug(info.getShortDescription()); assertTrue(p > 0); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java index 7ea4338..dfda854 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java @@ -341,10 +341,10 @@ public class TestBackupSystemTable { table.addWALFiles(files, "backup", "root"); - assertTrue(table.checkWALFile(files.get(0))); - assertTrue(table.checkWALFile(files.get(1))); - assertTrue(table.checkWALFile(files.get(2))); - assertFalse(table.checkWALFile(newFile)); + assertTrue(table.isWALFileDeletable(files.get(0))); + assertTrue(table.isWALFileDeletable(files.get(1))); + assertTrue(table.isWALFileDeletable(files.get(2))); + assertFalse(table.isWALFileDeletable(newFile)); cleanBackupTable(); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java index 0c0bf4a..3d37d07 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java @@ -1,14 +1,20 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package org.apache.hadoop.hbase.backup; import static org.junit.Assert.assertTrue; diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java index 983b850..1e23842 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java @@ -20,6 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.BackupAdmin; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; @@ -50,8 +51,8 @@ public class TestFullRestore extends TestBackupBase { TableName[] tableset = new TableName[] { table1 }; TableName[] tablemap = new TableName[] { table1_restore }; - RestoreClient client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap, false); + BackupAdmin client = getBackupAdmin(); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap, false)); HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); assertTrue(hba.tableExists(table1_restore)); TEST_UTIL.deleteTable(table1_restore); @@ -69,7 +70,7 @@ public class TestFullRestore extends TestBackupBase { LOG.info("backup complete"); assertTrue(checkSucceeded(backupId)); //restore [tableMapping] - String[] args = new String[]{"restore", BACKUP_ROOT_DIR, backupId, + String[] args = new String[]{BACKUP_ROOT_DIR, backupId, table1.getNameAsString(), table1_restore.getNameAsString() }; // Run backup int ret = ToolRunner.run(conf1, new RestoreDriver(), args); @@ -94,9 +95,9 @@ public class TestFullRestore extends TestBackupBase { TableName[] restore_tableset = new TableName[] { table2, table3 }; TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - RestoreClient client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupId, false, false, - restore_tableset, tablemap, false); + BackupAdmin client = getBackupAdmin(); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, false, + restore_tableset, tablemap, false)); HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); assertTrue(hba.tableExists(table2_restore)); assertTrue(hba.tableExists(table3_restore)); @@ -121,7 +122,7 @@ public class TestFullRestore extends TestBackupBase { //restore [tableMapping] - String[] args = new String[]{"restore", BACKUP_ROOT_DIR, backupId, + String[] args = new String[]{BACKUP_ROOT_DIR, backupId, StringUtils.join(restore_tableset, ","), StringUtils.join(tablemap, ",") }; // Run backup @@ -152,9 +153,9 @@ public class TestFullRestore extends TestBackupBase { LOG.info("backup complete"); TableName[] tableset = new TableName[] { table1 }; - RestoreClient client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupId, false, false, tableset, null, - true); + BackupAdmin client = getBackupAdmin(); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, false, tableset, null, + true)); } /** @@ -171,7 +172,7 @@ public class TestFullRestore extends TestBackupBase { LOG.info("backup complete"); TableName[] tableset = new TableName[] { table1 }; //restore [tableMapping] - String[] args = new String[]{"restore", BACKUP_ROOT_DIR, backupId, + String[] args = new String[]{BACKUP_ROOT_DIR, backupId, StringUtils.join(tableset, ","), "-overwrite" }; // Run restore int ret = ToolRunner.run(conf1, new RestoreDriver(), args); @@ -196,9 +197,9 @@ public class TestFullRestore extends TestBackupBase { assertTrue(checkSucceeded(backupId)); TableName[] restore_tableset = new TableName[] { table2, table3 }; - RestoreClient client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupId, false, - false, restore_tableset, null, true); + BackupAdmin client = getBackupAdmin(); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, + false, restore_tableset, null, true)); } /** @@ -215,7 +216,7 @@ public class TestFullRestore extends TestBackupBase { TableName[] restore_tableset = new TableName[] { table2, table3 }; //restore [tableMapping] - String[] args = new String[]{"restore", BACKUP_ROOT_DIR, backupId, + String[] args = new String[]{BACKUP_ROOT_DIR, backupId, StringUtils.join(restore_tableset, ","), "-overwrite" }; // Run backup int ret = ToolRunner.run(conf1, new RestoreDriver(), args); @@ -243,9 +244,9 @@ public class TestFullRestore extends TestBackupBase { TableName[] tableset = new TableName[] { TableName.valueOf("faketable") }; TableName[] tablemap = new TableName[] { table1_restore }; - RestoreClient client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap, - false); + BackupAdmin client = getBackupAdmin(); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap, + false)); } @@ -265,7 +266,7 @@ public class TestFullRestore extends TestBackupBase { TableName[] tableset = new TableName[] { TableName.valueOf("faketable") }; TableName[] tablemap = new TableName[] { table1_restore }; - String[] args = new String[]{"restore", BACKUP_ROOT_DIR, backupId, + String[] args = new String[]{BACKUP_ROOT_DIR, backupId, StringUtils.join(tableset, ","), StringUtils.join(tablemap, ",") }; // Run restore @@ -289,9 +290,9 @@ public class TestFullRestore extends TestBackupBase { TableName[] restore_tableset = new TableName[] { TableName.valueOf("faketable1"), TableName.valueOf("faketable2") }; TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - RestoreClient client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupId, false, - false, restore_tableset, tablemap, false); + BackupAdmin client = getBackupAdmin(); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, + false, restore_tableset, tablemap, false)); } /** @@ -310,7 +311,7 @@ public class TestFullRestore extends TestBackupBase { TableName[] restore_tableset = new TableName[] { TableName.valueOf("faketable1"), TableName.valueOf("faketable2") }; TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - String[] args = new String[]{"restore", BACKUP_ROOT_DIR, backupId, + String[] args = new String[]{BACKUP_ROOT_DIR, backupId, StringUtils.join(restore_tableset, ","), StringUtils.join(tablemap, ",") }; // Run restore diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java index 04ee015..988b156 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java @@ -24,11 +24,8 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.BackupAdmin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.HBaseAdmin; @@ -59,7 +56,7 @@ public class TestIncrementalBackup extends TestBackupBase { BackupRequest request = new BackupRequest(); request.setBackupType(BackupType.FULL).setTableList(tables).setTargetRootDir(BACKUP_ROOT_DIR); - String backupIdFull = admin.backupTables(request); + String backupIdFull = admin.getBackupAdmin().backupTables(request); assertTrue(checkSucceeded(backupIdFull)); @@ -91,7 +88,7 @@ public class TestIncrementalBackup extends TestBackupBase { request = new BackupRequest(); request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) .setTargetRootDir(BACKUP_ROOT_DIR); - String backupIdIncMultiple = admin.backupTables(request); + String backupIdIncMultiple = admin.getBackupAdmin().backupTables(request); assertTrue(checkSucceeded(backupIdIncMultiple)); // #4 - restore full backup for all tables, without overwrite @@ -101,10 +98,10 @@ public class TestIncrementalBackup extends TestBackupBase { TableName[] tablesMapFull = new TableName[] { table1_restore, table2_restore, table3_restore, table4_restore }; - RestoreClient client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupIdFull, false, false, + BackupAdmin client = getBackupAdmin(); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false, false, tablesRestoreFull, - tablesMapFull, false); + tablesMapFull, false)); // #5.1 - check tables for full restore HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin(); @@ -137,9 +134,8 @@ public class TestIncrementalBackup extends TestBackupBase { new TableName[] { table1, table2, table3 }; TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore, table3_restore }; - client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupIdIncMultiple, false, false, - tablesRestoreIncMultiple, tablesMapIncMultiple, true); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple, false, false, + tablesRestoreIncMultiple, tablesMapIncMultiple, true)); hTable = (HTable) conn.getTable(table1_restore); Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2)); @@ -159,16 +155,16 @@ public class TestIncrementalBackup extends TestBackupBase { request = new BackupRequest(); request.setBackupType(BackupType.INCREMENTAL).setTableList(tables) .setTargetRootDir(BACKUP_ROOT_DIR); - String backupIdIncEmpty = admin.backupTables(request); + String backupIdIncEmpty = admin.getBackupAdmin().backupTables(request); // #8 - restore incremental backup for single empty table, with overwrite TableName[] tablesRestoreIncEmpty = new TableName[] { table4 }; TableName[] tablesMapIncEmpty = new TableName[] { table4_restore }; - getRestoreClient().restore(BACKUP_ROOT_DIR, backupIdIncEmpty, false, false, + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncEmpty, false, false, tablesRestoreIncEmpty, - tablesMapIncEmpty, true); + tablesMapIncEmpty, true)); hTable = (HTable) conn.getTable(table4_restore); Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0)); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupNoDataLoss.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupNoDataLoss.java index b3cf4ee..c67f63f 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupNoDataLoss.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupNoDataLoss.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.BackupAdmin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.HTable; @@ -97,9 +98,9 @@ public class TestIncrementalBackupNoDataLoss extends TestBackupBase { TEST_UTIL.deleteTable(table2_restore); } - RestoreClient client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupIdInc1, false, true, tablesRestoreInc1, - tablesMapInc1, false); + BackupAdmin client = getBackupAdmin(); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupIdInc1, false, true, tablesRestoreInc1, + tablesMapInc1, false)); HTable hTable = (HTable) conn.getTable(table1_restore); Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2)); @@ -110,9 +111,8 @@ public class TestIncrementalBackupNoDataLoss extends TestBackupBase { TableName[] tablesRestoreInc2 = new TableName[] { table2 }; TableName[] tablesMapInc2 = new TableName[] { table2_restore }; - client = getRestoreClient(); - client.restore(BACKUP_ROOT_DIR, backupIdInc2, false, true, tablesRestoreInc2, - tablesMapInc2, false); + client.restore(createRestoreRequest(BACKUP_ROOT_DIR, backupIdInc2, false, true, tablesRestoreInc2, + tablesMapInc2, false)); hTable = (HTable) conn.getTable(table2_restore); Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5)); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java index 32a028c..f40ed9d 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java @@ -39,8 +39,8 @@ public class TestRemoteRestore extends TestBackupBase { LOG.info("backup complete"); TableName[] tableset = new TableName[] { table1 }; TableName[] tablemap = new TableName[] { table1_restore }; - getRestoreClient().restore(BACKUP_REMOTE_ROOT_DIR, backupId, false, false, tableset, - tablemap, false); + getBackupAdmin().restore(createRestoreRequest(BACKUP_REMOTE_ROOT_DIR, backupId, false, false, tableset, + tablemap, false)); HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); assertTrue(hba.tableExists(table1_restore)); TEST_UTIL.deleteTable(table1_restore); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java index 652a909..c86b4b4 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java @@ -46,8 +46,8 @@ public class TestRestoreBoundaryTests extends TestBackupBase { LOG.info("backup complete"); TableName[] tableset = new TableName[] { table1 }; TableName[] tablemap = new TableName[] { table1_restore }; - getRestoreClient().restore(BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap, - false); + getBackupAdmin().restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap, + false)); HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); assertTrue(hba.tableExists(table1_restore)); TEST_UTIL.deleteTable(table1_restore); @@ -65,9 +65,9 @@ public class TestRestoreBoundaryTests extends TestBackupBase { String backupId = fullTableBackup(tables); TableName[] restore_tableset = new TableName[] { table2, table3}; TableName[] tablemap = new TableName[] { table2_restore, table3_restore }; - getRestoreClient().restore(BACKUP_ROOT_DIR, backupId, false, false, restore_tableset, + getBackupAdmin().restore(createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, false, restore_tableset, tablemap, - false); + false)); HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); assertTrue(hba.tableExists(table2_restore)); assertTrue(hba.tableExists(table3_restore));