diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index d3b9812..881714f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import com.google.protobuf.CodedInputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -68,6 +69,8 @@ import org.apache.hadoop.hbase.util.Threads; public final class SnapshotManifest { private static final Log LOG = LogFactory.getLog(SnapshotManifest.class); + public static final String SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY = "snapshot.manifest.size.limit"; + private static final String DATA_MANIFEST_NAME = "data.manifest"; private List regionManifests; @@ -78,6 +81,7 @@ public final class SnapshotManifest { private final Configuration conf; private final Path workingDir; private final FileSystem fs; + private int manifestSizeLimit; private SnapshotManifest(final Configuration conf, final FileSystem fs, final Path workingDir, final SnapshotDescription desc, @@ -87,6 +91,8 @@ public final class SnapshotManifest { this.workingDir = workingDir; this.conf = conf; this.fs = fs; + + this.manifestSizeLimit = conf.getInt(SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 67108864); } /** @@ -512,7 +518,9 @@ public final class SnapshotManifest { FSDataInputStream in = null; try { in = fs.open(new Path(workingDir, DATA_MANIFEST_NAME)); - return SnapshotDataManifest.parseFrom(in); + CodedInputStream cin = CodedInputStream.newInstance(in); + cin.setSizeLimit(manifestSizeLimit); + return SnapshotDataManifest.parseFrom(cin); } catch (FileNotFoundException e) { return null; } finally { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index 96e3990..f739233 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -550,15 +550,30 @@ public class SnapshotTestingUtils { return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION); } + public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName, + final int numRegions) throws IOException { + return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION); + } + public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName) throws IOException { return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION); } + public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName, + final int numRegions) throws IOException { + return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION); + } + private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, final int version) throws IOException { + return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version); + } + + private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, + final int numRegions, final int version) throws IOException { HTableDescriptor htd = createHtd(tableName); - RegionData[] regions = createTable(htd, TEST_NUM_REGIONS); + RegionData[] regions = createTable(htd, numRegions); SnapshotDescription desc = SnapshotDescription.newBuilder() .setTable(htd.getNameAsString()) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java new file mode 100644 index 0000000..446e674 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotManifest.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +import static org.junit.Assert.fail; + +@Category({MasterTests.class, SmallTests.class}) +public class TestSnapshotManifest { + private final Log LOG = LogFactory.getLog(getClass()); + + private static final String tableName = "testSnapshotManifest"; + public final static int TEST_NUM_REGIONS = 200; + + private static HBaseTestingUtility TEST_UTIL; + private Configuration conf; + private FileSystem fs; + private Path rootDir; + private Path snapshotDir; + private SnapshotDescription snapshotDesc; + + @Before + public void setup() throws Exception { + TEST_UTIL = HBaseTestingUtility.createLocalHTU(); + + rootDir = TEST_UTIL.getDataTestDir(tableName); + fs = TEST_UTIL.getTestFileSystem(); + conf = TEST_UTIL.getConfiguration(); + + SnapshotTestingUtils.SnapshotMock snapshotMock = new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir); + SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = + snapshotMock.createSnapshotV2("snapshot", tableName, TEST_NUM_REGIONS); + for (int i = 0; i < TEST_NUM_REGIONS; i++) { + builder.addRegionV2(); + } + + snapshotDir = builder.commit(); + snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); + } + + @After + public void tearDown() throws Exception { + fs.delete(rootDir,true); + } + + @Test + public void testReadSnapshotManifest() throws IOException { + try { + conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 1024); + SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); + fail("fail to test snapshot manifest because message size is too small."); + } catch (InvalidProtocolBufferException ipbe) { + try { + conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 130023424); + SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); + LOG.info("open snapshot manifest succeed."); + } catch (InvalidProtocolBufferException ipbe2) { + fail("fail to take snapshot because Manifest proto-message too large."); + } + } + } +}