diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index 38ccf08..a38e7fc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import com.google.protobuf.CodedInputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -64,7 +65,8 @@ import org.apache.hadoop.hbase.util.Threads; public class SnapshotManifest { private static final Log LOG = LogFactory.getLog(SnapshotManifest.class); - private static final String DATA_MANIFEST_NAME = "data.manifest"; + public static final String DATA_MANIFEST_NAME = "data.manifest"; + private static final String CONF_SNAPSHOT_MANIFEST_SIZE_LIMIT = "snapshot.manifest.size.limit"; private List regionManifests; private SnapshotDescription desc; @@ -74,6 +76,7 @@ public class SnapshotManifest { private final Configuration conf; private final Path workingDir; private final FileSystem fs; + private int manifestSizeLimit; private SnapshotManifest(final Configuration conf, final FileSystem fs, final Path workingDir, final SnapshotDescription desc, @@ -83,6 +86,8 @@ public class SnapshotManifest { this.workingDir = workingDir; this.conf = conf; this.fs = fs; + + this.manifestSizeLimit = conf.getInt(CONF_SNAPSHOT_MANIFEST_SIZE_LIMIT, 67108864); } /** @@ -430,7 +435,9 @@ public class SnapshotManifest { FSDataInputStream in = null; try { in = fs.open(new Path(workingDir, DATA_MANIFEST_NAME)); - return SnapshotDataManifest.parseFrom(in); + CodedInputStream cin = CodedInputStream.newInstance(in); + cin.setSizeLimit(manifestSizeLimit); + return SnapshotDataManifest.parseFrom(cin); } catch (FileNotFoundException e) { return null; } finally { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 9c1e42c..37fa561 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -1695,6 +1695,33 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false); return createLocalHRegion(info, htd, hlog); } + + /** + * @param tableName + * @param startKey + * @param stopKey + * @param callingMethod + * @param conf + * @param isReadOnly + * @param families + * @throws IOException + * @return A regioninfo on which you must call + * {@link HRegionInfo} when done. + */ + public HRegionInfo createHRegionInfo(byte[] tableName, byte[] startKey, byte[] stopKey, + String callingMethod, Configuration conf, boolean isReadOnly, Durability durability, + HLog hlog, byte[]... families) throws IOException { + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.setReadOnly(isReadOnly); + for (byte[] family : families) { + HColumnDescriptor hcd = new HColumnDescriptor(family); + // Set default to be three versions. + hcd.setMaxVersions(Integer.MAX_VALUE); + htd.addFamily(hcd); + } + htd.setDurability(durability); + return new HRegionInfo(htd.getTableName(), startKey, stopKey, false); + } // // ========================================================================== diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java index e69de29..74ce0bf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +import static org.junit.Assert.fail; + +@Category(LargeTests.class) +public class TestSnapshotManifest { + + private final Log LOG = LogFactory.getLog(getClass()); + + private static final String tableName = "testSnapshotManifest"; + public final static int TEST_NUM_REGIONS = 2000000; + + private static HBaseTestingUtility TEST_UTIL; + private Configuration conf; + private FileSystem fs; + private Path rootDir; + private Path snapshotDir; + private HBaseProtos.SnapshotDescription snapshotDesc; + + @Before + public void setup() throws Exception { + + TEST_UTIL = HBaseTestingUtility.createLocalHTU(); + + rootDir = TEST_UTIL.getDataTestDir(tableName); + fs = TEST_UTIL.getTestFileSystem(); + conf = TEST_UTIL.getConfiguration(); + FSUtils.setRootDir(conf, rootDir); + + + SnapshotTestingUtils.SnapshotMock snapshotMock = new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir); + SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName); + snapshotDir = builder.commit(); + snapshotDesc = builder.getSnapshotDescription(); + + SnapshotProtos.SnapshotDataManifest.Builder dataManifestBuilder = SnapshotProtos.SnapshotDataManifest.newBuilder(); + + HBaseProtos.TableName.Builder protoTableNameBuilder = HBaseProtos.TableName.newBuilder(); + protoTableNameBuilder.setNamespace(ByteString.copyFrom("default".getBytes())); + protoTableNameBuilder.setQualifier(ByteString.copyFrom(tableName.getBytes())); + HBaseProtos.TableName protoTableName = protoTableNameBuilder.build(); + + for(int i = 0; i < TEST_NUM_REGIONS; i++) { + SnapshotProtos.SnapshotRegionManifest.Builder dataRegionManifestBuilder = SnapshotProtos.SnapshotRegionManifest.newBuilder(); + + HRegionInfo regionInfo = TEST_UTIL.createHRegionInfo(Bytes.toBytes(tableName), + null, null, tableName, conf, false, Durability.SYNC_WAL, null, Bytes.toBytes("cf")); + + dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo)); + dataManifestBuilder.addRegionManifests(dataRegionManifestBuilder.build()); + } + + HBaseProtos.TableSchema.Builder talbeSchemaBuilder = HBaseProtos.TableSchema.newBuilder(); + talbeSchemaBuilder.setTableName(protoTableName); + dataManifestBuilder.setTableSchema(talbeSchemaBuilder.build()); + + SnapshotProtos.SnapshotDataManifest dataManifest = dataManifestBuilder.build(); + writeDataManifest(dataManifest); + + + } + + @After + public void tearDown() throws Exception { + + fs.delete(rootDir,true); + } + + @Test + public void testReadSnapshotManifest() throws IOException { + + try { + SnapshotManifest.open(conf, fs, rootDir, snapshotDesc); + fail("fail to test snapshot manifest because message size is too small."); + }catch(InvalidProtocolBufferException ipbe) { + try { + conf.setInt("snapshot.manifest.size.limit", 130023424); + SnapshotManifest.open(conf, fs, rootDir, snapshotDesc); + LOG.info("take snapshot successfully."); + }catch(InvalidProtocolBufferException ipbe2) { + fail("fail to take snapshot because Manifest proto-message too large."); + } + } + } + + private void writeDataManifest(final SnapshotProtos.SnapshotDataManifest manifest) + throws IOException { + FSDataOutputStream stream = fs.create(new Path(rootDir, SnapshotManifest.DATA_MANIFEST_NAME)); + try { + manifest.writeTo(stream); + } finally { + stream.close(); + } + } +}