From a0841035125dda128c0e414275a18074ae09f3cf Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Wed, 24 Dec 2014 11:28:37 +0800 Subject: [PATCH] Update mini cluster test for query combination test --- .../util/HBaseMiniclusterMetadataTestCase.java | 64 +++++---- .../java/com/kylinolap/job/ImportHBaseData.java | 155 +++++++++++++++++++++ .../com/kylinolap/query/test/KylinQueryTest.java | 4 +- .../com/kylinolap/storage/test/StorageTest.java | 2 + 4 files changed, 190 insertions(+), 35 deletions(-) create mode 100644 job/src/test/java/com/kylinolap/job/ImportHBaseData.java diff --git a/common/src/main/java/com/kylinolap/common/util/HBaseMiniclusterMetadataTestCase.java b/common/src/main/java/com/kylinolap/common/util/HBaseMiniclusterMetadataTestCase.java index e5191ac..01678c4 100644 --- a/common/src/main/java/com/kylinolap/common/util/HBaseMiniclusterMetadataTestCase.java +++ b/common/src/main/java/com/kylinolap/common/util/HBaseMiniclusterMetadataTestCase.java @@ -77,17 +77,14 @@ public void createTestMetadata() { staticCreateTestMetadata(MINICLUSTER_TEST_DATA); // Overwrite the hbase url with the minicluster's - KylinConfig.getInstanceFromEnv().setMetadataUrl("kylin_metadata_qa@" + hbaseconnectionUrl); - KylinConfig.getInstanceFromEnv().setStorageUrl(hbaseconnectionUrl); + updateKylinConfigWithMinicluster(); } /** * Start the minicluster; Sub-classes should invoke this in BeforeClass method. - * @throws IOException - * @throws ClassNotFoundException - * @throws InterruptedException + * @throws Exception */ - public static void startupMinicluster() throws IOException, ClassNotFoundException, InterruptedException { + public static void startupMinicluster() throws Exception { staticCreateTestMetadata(MINICLUSTER_TEST_DATA); if (!clusterStarted) { @@ -97,43 +94,44 @@ public static void startupMinicluster() throws IOException, ClassNotFoundExcepti clusterStarted = true; } } + } else { + updateKylinConfigWithMinicluster(); } } - private static void startupMiniClusterAndImportData() { + private static void updateKylinConfigWithMinicluster() { + + KylinConfig.getInstanceFromEnv().setMetadataUrl("kylin_metadata_qa@" + hbaseconnectionUrl); + KylinConfig.getInstanceFromEnv().setStorageUrl(hbaseconnectionUrl); + } + + private static void startupMiniClusterAndImportData() throws Exception { System.out.println("Going to start mini cluster."); - try { - hbaseCluster = UTIL.startMiniCluster(); + hbaseCluster = UTIL.startMiniCluster(); - UTIL.startMiniMapReduceCluster(); - config = hbaseCluster.getConf(); - String host = config.get(HConstants.ZOOKEEPER_QUORUM); - String port = config.get(HConstants.ZOOKEEPER_CLIENT_PORT); - String parent = config.get(HConstants.ZOOKEEPER_ZNODE_PARENT); + UTIL.startMiniMapReduceCluster(); + config = hbaseCluster.getConf(); + String host = config.get(HConstants.ZOOKEEPER_QUORUM); + String port = config.get(HConstants.ZOOKEEPER_CLIENT_PORT); + String parent = config.get(HConstants.ZOOKEEPER_ZNODE_PARENT); - // reduce rpc retry - config.set(HConstants.HBASE_CLIENT_PAUSE, "3000"); - config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5"); - config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000"); + // reduce rpc retry + config.set(HConstants.HBASE_CLIENT_PAUSE, "3000"); + config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5"); + config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000"); - hbaseconnectionUrl = "hbase:" + host + ":" + port + ":" + parent; + hbaseconnectionUrl = "hbase:" + host + ":" + port + ":" + parent; - KylinConfig.getInstanceFromEnv().setMetadataUrl("kylin_metadata_qa@" + hbaseconnectionUrl); - KylinConfig.getInstanceFromEnv().setStorageUrl(hbaseconnectionUrl); - - // create the metadata htables; - HBaseResourceStore store = new HBaseResourceStore(KylinConfig.getInstanceFromEnv()); + updateKylinConfigWithMinicluster(); + // create the metadata htables; + HBaseResourceStore store = new HBaseResourceStore(KylinConfig.getInstanceFromEnv()); + + // import the table content + importHBaseData(true, true); - // import the table content - importHBaseData(true, true); - } catch (Exception e) { - e.printStackTrace(); - } - } - - + public static void importHBaseData(boolean importMetadataTables, boolean importCubeTables) throws IOException, ClassNotFoundException, InterruptedException { if (!importMetadataTables && !importCubeTables) @@ -174,7 +172,7 @@ public static void importHBaseData(boolean importMetadataTables, boolean importC for (String table : tableNames) { - if (!(table.startsWith("kylin_metadata_qa") && importMetadataTables || table.startsWith("KYLIN_") && importCubeTables)) { + if (!(table.equalsIgnoreCase("kylin_metadata_qa") && importMetadataTables || table.startsWith("KYLIN_") && importCubeTables)) { continue; } diff --git a/job/src/test/java/com/kylinolap/job/ImportHBaseData.java b/job/src/test/java/com/kylinolap/job/ImportHBaseData.java new file mode 100644 index 0000000..d683ac3 --- /dev/null +++ b/job/src/test/java/com/kylinolap/job/ImportHBaseData.java @@ -0,0 +1,155 @@ +package com.kylinolap.job; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; + +import com.kylinolap.common.KylinConfig; +import com.kylinolap.common.persistence.HBaseConnection; +import com.kylinolap.common.persistence.HBaseResourceStore; +import com.kylinolap.common.util.AbstractKylinTestCase; +import com.kylinolap.common.util.CliCommandExecutor; +import com.kylinolap.common.util.SSHClient; +import com.kylinolap.common.util.TarGZUtil; + +public class ImportHBaseData { + + KylinConfig kylinConfig; + HTableDescriptor[] allTables; + Configuration config; + HBaseAdmin hbase; + CliCommandExecutor cli = null; + String importFolder = "/tmp/hbase-export/"; + String backupArchive = null; + String tableNameBase; + + public void setup() throws IOException { + + KylinConfig.destoryInstance(); + System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA); + + kylinConfig = KylinConfig.getInstanceFromEnv(); + cli = kylinConfig.getCliCommandExecutor(); + + String metadataUrl = kylinConfig.getMetadataUrl(); + // split TABLE@HBASE_URL + int cut = metadataUrl.indexOf('@'); + tableNameBase = metadataUrl.substring(0, cut); + String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); + + HConnection conn = HBaseConnection.get(hbaseUrl); + try { + hbase = new HBaseAdmin(conn); + config = hbase.getConfiguration(); + //allTables = hbase.listTables(); + } catch (IOException e) { + e.printStackTrace(); + throw e; + } + + uploadTarballToRemote(); + } + + public void tearDown() { + // cleanup sandbox disk + try { + if (cli != null && importFolder != null) { + cli.execute("rm -r " + importFolder); + } + } catch (IOException e) { + e.printStackTrace(); + } + + KylinConfig.destoryInstance(); + + } + + public void importTables() throws IOException { + // create the metadata htables; + HBaseResourceStore store = new HBaseResourceStore(KylinConfig.getInstanceFromEnv()); + + List tablelocations = getTablesBackupLocations(importFolder); + for (String tableLocation : tablelocations) { + String table = tableLocation.substring(tableLocation.lastIndexOf("/") + 1); + + if (!(table.equalsIgnoreCase("kylin_metadata_qa") || table.startsWith("KYLIN_"))) { + continue; + } + + if (table.startsWith("KYLIN_")) { + // create the cube table; otherwise the import will fail. + HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "F1", "F2"); + } + cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Import " + table + " file://" + tableLocation); + } + + } + + public void uploadTarballToRemote() throws IOException { + + cli.execute("mkdir -p /tmp/hbase-export/"); + SSHClient ssh = new SSHClient(kylinConfig.getRemoteHadoopCliHostname(), kylinConfig.getRemoteHadoopCliUsername(), kylinConfig.getRemoteHadoopCliPassword()); + try { + // ssh.scpFileToRemote("../examples/test_case_data/minicluster/hbase-export.tar.gz", importFolder); + } catch (Exception e) { + e.printStackTrace(); + } + + cli.execute("tar -xzf /tmp/hbase-export/hbase-export.tar.gz --directory=" + importFolder); + } + + private List getTablesBackupLocations(String exportBase) throws IOException { + File exportFile = new File("../examples/test_case_data/minicluster/hbase-export.tar.gz"); + + if (!exportFile.exists()) { + return null; + } + + File folder = new File("/tmp/hbase-export/"); + + if (folder.exists()) { + folder.delete(); + } + + folder.mkdirs(); + folder.deleteOnExit(); + + TarGZUtil.uncompressTarGZ(exportFile, folder); + + String[] child = folder.list(); + + assert child.length == 1; + + String backupTime = child[0]; + + File backupFolder = new File(folder, backupTime); + + String[] tableNames = backupFolder.list(); + + List locations = new ArrayList(15); + + for(String t: tableNames) { + locations.add(exportBase + backupTime + "/" + t); + } + + return locations; + } + + public static void main(String[] args) { + ImportHBaseData export = new ImportHBaseData(); + try { + export.setup(); + export.importTables(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + //export.tearDown(); + } + } +} diff --git a/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java b/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java index c5e8247..bb5022b 100644 --- a/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java +++ b/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java @@ -56,13 +56,13 @@ public static void setUp() throws Exception { preferCubeOf(joinType); } - protected static void setupAll() throws SQLException, IOException, ClassNotFoundException, InterruptedException { + protected static void setupAll() throws Exception { setUpEnv(); setUpCubeConn(); setUpH2Conn(); } - private static void setUpEnv() throws IOException, ClassNotFoundException, InterruptedException { + private static void setUpEnv() throws Exception { String queryUseMinicluster = System.getProperty("queryUseMinicluster"); diff --git a/storage/src/test/java/com/kylinolap/storage/test/StorageTest.java b/storage/src/test/java/com/kylinolap/storage/test/StorageTest.java index b45b5f9..05ab5ce 100644 --- a/storage/src/test/java/com/kylinolap/storage/test/StorageTest.java +++ b/storage/src/test/java/com/kylinolap/storage/test/StorageTest.java @@ -22,6 +22,7 @@ import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; @@ -71,6 +72,7 @@ public void setUp() throws Exception { CubeManager cubeMgr = CubeManager.getInstance(this.getTestConfig()); cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY"); + Assert.assertNotNull(cube); storageEngine = StorageEngineFactory.getStorageEngine(cube); String url = KylinConfig.getInstanceFromEnv().getStorageUrl(); context = new StorageContext();