diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestScanMobSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestScanMobSnapshot.java new file mode 100644 index 0000000..22d1ab8 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestScanMobSnapshot.java @@ -0,0 +1,232 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.snapshot; + +import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.mob.MobConstants; +import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test scan the cloned table when deleting the original table + */ +@Category(LargeTests.class) +public class TestScanMobSnapshot { + + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final byte[] FAMILY = Bytes.toBytes("cf"); + private final byte[] STARTROW = Bytes.toBytes(START_KEY); + + private byte[] snapshotName; + private int snapshotRows; + private TableName tableName; + private Admin admin; + private HTable table; + private HTable clonedTable; + private HColumnDescriptor hcd; + private HTableDescriptor desc; + private Random random = new Random(); + private List expected = new ArrayList(); + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); + TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); + TEST_UTIL.getConfiguration().setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); + TEST_UTIL.startMiniCluster(3); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * Initialize the tests with create a table filled with some data and take a snapshot + */ + @Before + public void setup() throws Exception { + this.admin = TEST_UTIL.getHBaseAdmin(); + long tid = System.currentTimeMillis(); + tableName = TableName.valueOf("testtb-" + tid); + snapshotName = Bytes.toBytes("snaptb-" + tid); + // Create Table + createTable(); + table = new HTable(TEST_UTIL.getConfiguration(), tableName.getNameAsString()); + try { + // Insert data + loadData(10); + snapshotRows = MobSnapshotTestingUtils.countMobRows(table); + // Take a snapshot + admin.snapshot(snapshotName, tableName); + } finally { + table.close(); + } + } + + /** + * Create a Mob table + */ + private void createTable() throws MasterNotRunningException, ZooKeeperConnectionException, + IOException { + desc = new HTableDescriptor(tableName); + hcd = new HColumnDescriptor(FAMILY); + hcd.setMobEnabled(true); + hcd.setMobThreshold(0); + hcd.setMaxVersions(4); + desc.addFamily(hcd); + admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + admin.createTable(desc); + } + + /** + * Load some data to the Mob table + */ + private void loadData(int num) throws IOException, InterruptedException { + for (int i = 0; i < num; i++) { + Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(i))); + byte[] dummyData = generateMobValue(100); + p.add(FAMILY, Bytes.toBytes("colX"), dummyData); + expected.add(Bytes.toString(dummyData)); + table.put(p); + } + table.flushCommits(); + admin.flush(tableName); + } + + /** + * Generate the mob value. + * @param size the size of the value + * @return the mob value generated + */ + private byte[] generateMobValue(int size) { + byte[] mobVal = new byte[size]; + random.nextBytes(mobVal); + return mobVal; + } + + /** + * Get the file numbers in the path + */ + protected int getFileNum(Path path) throws FileNotFoundException, IOException { + FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); + int num = 0; + if (fs.exists(path)) { + for(FileStatus fileStatus : TEST_UTIL.getTestFileSystem().listStatus(path)) { + if(fileStatus.isFile()) { + num++; + } + } + return num; + } else { + return 0; + } + } + + @Test + public void testScanClonedTableWhenDeleteOriginalTable() throws IOException, + InterruptedException { + + TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis()); + // Clone the table + admin.cloneSnapshot(snapshotName, clonedTableName); + + // Get the number of hfiles in the mob dir + Path mobDirPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, + Bytes.toString(FAMILY)); + int mobFiles = getFileNum(mobDirPath); + + // Scan the cloned Table + Scan scan = new Scan(); + scan.setCaching(1); + clonedTable = new HTable(TEST_UTIL.getConfiguration(), clonedTableName); + ResultScanner results = clonedTable.getScanner(scan); + int count = 0; + for (Result res : results) { + List cells = res.listCells(); + for (Cell cell : cells) { + // Verify the value + Assert.assertEquals(expected.get(count), Bytes.toString(CellUtil.cloneValue(cell))); + count++; + // Delete the table after having read one cell. + if (count == 1) { + // Delete the orignal table + TEST_UTIL.deleteTable(tableName); + // Get the number of the hfiles in archive dir + Path mobArchivePath = HFileArchiveUtil.getStoreArchivePath(TEST_UTIL.getConfiguration(), + tableName, MobUtils.getMobRegionInfo(tableName).getEncodedName(), + Bytes.toString(FAMILY)); + // Verify the table archive success + Assert.assertEquals(mobFiles, getFileNum(mobArchivePath)); + Assert.assertEquals(0, getFileNum(mobDirPath)); + } + } + } + results.close(); + clonedTable.close(); + Assert.assertEquals(snapshotRows, count); + // Delete the cloned table + TEST_UTIL.deleteTable(clonedTableName); + } +}