From 95bc44068e448d11a0fbcb42e5e9d4d5ac735a1c Mon Sep 17 00:00:00 2001 From: huzheng Date: Tue, 20 Nov 2018 00:16:12 +0800 Subject: [PATCH] UT --- .../snapshot/TestSnapshotWhenChoreCleaning.java | 149 +++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotWhenChoreCleaning.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotWhenChoreCleaning.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotWhenChoreCleaning.java new file mode 100644 index 0000000..804a1bb --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotWhenChoreCleaning.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TestTableName; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Test Case for HBASE-21387 + */ +@Category({ MediumTests.class }) +public class TestSnapshotWhenChoreCleaning { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule + .forClass(TestSnapshotWhenChoreCleaning.class); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Configuration CONF = TEST_UTIL.getConfiguration(); + private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotClientRetries.class); + private static final TableName TABLE_NAME = TableName.valueOf("testTable"); + private static final int MAX_SPLIT_KEYS_NUM = 1000; + private static final byte[] FAMILY = Bytes.toBytes("family"); + private static final byte[] QUALIFIER = Bytes.toBytes("qualifier"); + private static final byte[] VALUE = Bytes.toBytes("value"); + private static Table TABLE; + + @Rule + public TestTableName TEST_TABLE = new TestTableName(); + + @BeforeClass + public static void setUp() throws Exception { + // Set the hbase.snapshot.thread.pool.max to 1; + CONF.setInt("hbase.snapshot.thread.pool.max", 1); + // Enable snapshot + CONF.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + // Start MiniCluster. + TEST_UTIL.startMiniCluster(1); + // Create talbe + createTable(); + } + + private static byte[] integerToBytes(int i) { + return Bytes.toBytes(String.format("%06d", i)); + } + + private static void createTable() throws IOException { + byte[][] splitKeys = new byte[MAX_SPLIT_KEYS_NUM][]; + for (int i = 0; i < splitKeys.length; i++) { + splitKeys[i] = integerToBytes(i); + } + TABLE = TEST_UTIL.createTable(TABLE_NAME, FAMILY, splitKeys); + } + + @AfterClass + public static void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + private static void loadDataAndFlush() throws IOException { + for (int i = 0; i < MAX_SPLIT_KEYS_NUM; i++) { + Put put = + new Put(integerToBytes(i)).addColumn(FAMILY, QUALIFIER, + Bytes.add(VALUE, Bytes.toBytes(i))); + TABLE.put(put); + } + TEST_UTIL.flush(TABLE_NAME); + } + + @Test + public void testSnapshotWhenSnapshotHFileCleanerRunning() throws Exception { + // Load data and flush to generate huge number of HFiles. + loadDataAndFlush(); + + SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner(); + cleaner.init(ImmutableMap.of(HMaster.MASTER, TEST_UTIL.getHBaseCluster().getMaster())); + + FileSystem fs = FSUtils.getCurrentFileSystem(CONF); + List fileNames = + SnapshotTestingUtils.listHFileNames(fs, + FSUtils.getTableDir(FSUtils.getRootDir(CONF), TABLE_NAME)); + List files = new ArrayList<>(); + for (String fileName : fileNames) { + files.add(fs.getFileStatus(new Path(fileName))); + } + + for (int i = 0; i < 3; i++) { + final String snapshotName = "snapshot-" + i; + Runnable runnable = () -> { + try { + TEST_UTIL.getAdmin().snapshot(snapshotName, TABLE_NAME); + } catch (IOException e) { + LOG.error("Snapshot failed: ", e); + } + }; + Thread t = new Thread(runnable); + t.start(); + Iterable toDeleteFiles = cleaner.getDeletableFiles(files); + t.join(); + Assert.assertEquals(Lists.newArrayList(toDeleteFiles).size(), 0); + TEST_UTIL.getAdmin().deleteSnapshot(snapshotName); + } + } +} -- 2.3.2 (Apple Git-55)