From a675099aba54509ccb74f8cce2c629277dbeb10d Mon Sep 17 00:00:00 2001 From: "Shao feng, Shi" Date: Thu, 27 Nov 2014 16:25:00 +0800 Subject: [PATCH] Add files/changes for running test cases in mini clusters --- .../java/com/kylinolap/common/KylinConfig.java | 5 + .../common/util/HBaseMetadataTestCase.java | 6 +- .../minicluster/capacity-scheduler.xml | 111 +++++++++++ examples/test_case_data/minicluster/core-site.xml | 8 + examples/test_case_data/minicluster/hdfs-site.xml | 8 + .../test_case_data/minicluster/kylin.properties | 72 ++++++++ .../minicluster/kylin_job_conf.lzo_disabled.xml | 60 ++++++ .../minicluster/kylin_job_conf.lzo_enabled.xml | 60 ++++++ .../test_case_data/minicluster/kylin_job_conf.xml | 60 ++++++ .../test_case_data/minicluster/mapred-site.xml | 8 + job/pom.xml | 16 ++ .../main/java/com/kylinolap/job/cmd/ShellCmd.java | 2 +- .../kylinolap/job/MiniClusterMetadataTestCase.java | 84 +++++++++ .../job/hadoop/hive/JoinedFlatTableTest.java | 4 +- .../com/kylinolap/query/test/KylinQueryTest.java | 3 +- .../kylinolap/rest/service/ServiceTestBase.java | 3 +- storage/pom.xml | 11 ++ .../storage/minicluster/HiveJDBCClientTest.java | 169 +++++++++++++++++ .../storage/minicluster/HiveMiniClusterTest.java | 117 ++++++++++++ .../storage/minicluster/HiveMiniClusterTest2.java | 126 +++++++++++++ .../storage/minicluster/HiveMiniClusterTest3.java | 205 +++++++++++++++++++++ 21 files changed, 1130 insertions(+), 8 deletions(-) create mode 100644 examples/test_case_data/minicluster/capacity-scheduler.xml create mode 100644 examples/test_case_data/minicluster/core-site.xml create mode 100644 examples/test_case_data/minicluster/hdfs-site.xml create mode 100644 examples/test_case_data/minicluster/kylin.properties create mode 100644 examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml create mode 100644 examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml create mode 100644 examples/test_case_data/minicluster/kylin_job_conf.xml create mode 100644 examples/test_case_data/minicluster/mapred-site.xml create mode 100644 job/src/test/java/com/kylinolap/job/MiniClusterMetadataTestCase.java create mode 100644 storage/src/test/java/com/kylinolap/storage/minicluster/HiveJDBCClientTest.java create mode 100644 storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest.java create mode 100644 storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest2.java create mode 100644 storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest3.java diff --git a/common/src/main/java/com/kylinolap/common/KylinConfig.java b/common/src/main/java/com/kylinolap/common/KylinConfig.java index 2e834d5..98fc98d 100644 --- a/common/src/main/java/com/kylinolap/common/KylinConfig.java +++ b/common/src/main/java/com/kylinolap/common/KylinConfig.java @@ -577,6 +577,11 @@ public String getMetadataUrlPrefix() { public void setMetadataUrl(String metadataUrl) { kylinConfig.setProperty(KYLIN_METADATA_URL, metadataUrl); } + + public void setStorageUrl(String storageUrl) { + kylinConfig.setProperty(KYLIN_STORAGE_URL, storageUrl); + } + /** * return -1 if there is no setting diff --git a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java index a1dda9a..0cd4ada 100644 --- a/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java +++ b/common/src/main/java/com/kylinolap/common/util/HBaseMetadataTestCase.java @@ -23,12 +23,12 @@ */ public class HBaseMetadataTestCase extends AbstractKylinTestCase { - public static void staticCreateTestMetadata() { + public static void staticCreateTestMetadata(String kylinConfigFolder) { KylinConfig.destoryInstance(); if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null) - System.setProperty(KylinConfig.KYLIN_CONF, SANDBOX_TEST_DATA); + System.setProperty(KylinConfig.KYLIN_CONF, kylinConfigFolder); } @@ -39,7 +39,7 @@ public static void staticCleanupTestMetadata() { @Override public void createTestMetadata() { - staticCreateTestMetadata(); + staticCreateTestMetadata(SANDBOX_TEST_DATA); } @Override diff --git a/examples/test_case_data/minicluster/capacity-scheduler.xml b/examples/test_case_data/minicluster/capacity-scheduler.xml new file mode 100644 index 0000000..cddca6c --- /dev/null +++ b/examples/test_case_data/minicluster/capacity-scheduler.xml @@ -0,0 +1,111 @@ + + + + + yarn.scheduler.capacity.maximum-applications + 10000 + + Maximum number of applications that can be pending and running. + + + + + yarn.scheduler.capacity.maximum-am-resource-percent + 0.1 + + Maximum percent of resources in the cluster which can be used to run + application masters i.e. controls number of concurrent running + applications. + + + + + yarn.scheduler.capacity.resource-calculator + org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator + + The ResourceCalculator implementation to be used to compare + Resources in the scheduler. + The default i.e. DefaultResourceCalculator only uses Memory while + DominantResourceCalculator uses dominant-resource to compare + multi-dimensional resources such as Memory, CPU etc. + + + + + yarn.scheduler.capacity.root.queues + default + + The queues at the this level (root is the root queue). + + + + + yarn.scheduler.capacity.root.default.capacity + 100 + Default queue target capacity. + + + + yarn.scheduler.capacity.root.default.user-limit-factor + 1 + + Default queue user limit a percentage from 0.0 to 1.0. + + + + + yarn.scheduler.capacity.root.default.maximum-capacity + 100 + + The maximum capacity of the default queue. + + + + + yarn.scheduler.capacity.root.default.state + RUNNING + + The state of the default queue. State can be one of RUNNING or STOPPED. + + + + + yarn.scheduler.capacity.root.default.acl_submit_applications + * + + The ACL of who can submit jobs to the default queue. + + + + + yarn.scheduler.capacity.root.default.acl_administer_queue + * + + The ACL of who can administer jobs on the default queue. + + + + + yarn.scheduler.capacity.node-locality-delay + -1 + + Number of missed scheduling opportunities after which the CapacityScheduler + attempts to schedule rack-local containers. + Typically this should be set to number of racks in the cluster, this + feature is disabled by default, set to -1. + + + + diff --git a/examples/test_case_data/minicluster/core-site.xml b/examples/test_case_data/minicluster/core-site.xml new file mode 100644 index 0000000..950bcd0 --- /dev/null +++ b/examples/test_case_data/minicluster/core-site.xml @@ -0,0 +1,8 @@ + + + + + fs.default.name + hdfs://localhost:9010 + + \ No newline at end of file diff --git a/examples/test_case_data/minicluster/hdfs-site.xml b/examples/test_case_data/minicluster/hdfs-site.xml new file mode 100644 index 0000000..2667860 --- /dev/null +++ b/examples/test_case_data/minicluster/hdfs-site.xml @@ -0,0 +1,8 @@ + + + + + dfs.replication + 1 + + \ No newline at end of file diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties new file mode 100644 index 0000000..a025530 --- /dev/null +++ b/examples/test_case_data/minicluster/kylin.properties @@ -0,0 +1,72 @@ +## Config for Kylin Engine ## + +# List of web servers in use, this enables one web server instance to sync up with other servers. +kylin.rest.servers=ADMIN:KYLIN@localhost + +# The metadata store in hbase +kylin.metadata.url=kylin_metadata_qa@hbase:minicluster:/hbase-unsecure + +# The storage for final cube file in hbase +kylin.storage.url=hbase:minicluster:/hbase-unsecure + +# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase +kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar + +# Temp folder in hdfs +kylin.hdfs.working.dir=/tmp + +# Path to the local(relative to job engine) job jar, job engine will use this jar +kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar + +kylin.job.mapreduce.default.reduce.input.mb=500 + +# If true, job engine will not assume that hadoop CLI reside on the same server as it self +# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password +kylin.job.run.as.remote.cmd=false + +# Only necessary when kylin.job.run.as.remote.cmd=true +kylin.job.remote.cli.hostname= + +# Only necessary when kylin.job.run.as.remote.cmd=true +kylin.job.remote.cli.username= + +# Only necessary when kylin.job.run.as.remote.cmd=true +kylin.job.remote.cli.password= + +# Hive JDBC connection properties; Not in use now +kylin.hive.jdbc.driver=org.apache.hive.jdbc.HiveDriver +kylin.hive.jdbc.url=jdbc:hive2:/// +kylin.hive.jdbc.user= +kylin.hive.jdbc.password= + +# Used by test cases to prepare synthetic data for sample cube +kylin.job.remote.cli.working.dir=/tmp/kylin + +# Use hive to flatten star schema tables +kylin.job.hive.flatten=true + +# Max count of concurrent jobs running +kylin.job.concurrent.max.limit=10 + +# The url to check hadoop job status +kylin.job.yarn.app.rest.check.status.url=http://127.0.0.1:8088/ws/v1/cluster/apps/${job_id}?anonymous=true + +# Time interval to check hadoop job status +kylin.job.yarn.app.rest.check.interval.seconds=10 + + +## Config for Restful APP ## +# database connection settings: +ldap.server= +ldap.username= +ldap.password= +ldap.user.searchBase= +ldap.user.searchPattern= +ldap.user.groupSearchBase= +ldap.service.searchBase=OU= +ldap.service.searchPattern= +ldap.service.groupSearchBase= +acl.adminRole= +acl.defaultRole= +ganglia.group= +ganglia.port=8664 \ No newline at end of file diff --git a/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml b/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml new file mode 100644 index 0000000..f62ce55 --- /dev/null +++ b/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml @@ -0,0 +1,60 @@ + + + + + mapreduce.job.split.metainfo.maxsize + -1 + The maximum permissible size of the split metainfo file. + The JobTracker won't attempt to read split metainfo files bigger than + the configured value. No limits if set to -1. + + + + + + + mapreduce.job.max.split.locations + 2000 + No description + + + + dfs.replication + 2 + Block replication + + \ No newline at end of file diff --git a/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml b/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml new file mode 100644 index 0000000..f62ce55 --- /dev/null +++ b/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml @@ -0,0 +1,60 @@ + + + + + mapreduce.job.split.metainfo.maxsize + -1 + The maximum permissible size of the split metainfo file. + The JobTracker won't attempt to read split metainfo files bigger than + the configured value. No limits if set to -1. + + + + + + + mapreduce.job.max.split.locations + 2000 + No description + + + + dfs.replication + 2 + Block replication + + \ No newline at end of file diff --git a/examples/test_case_data/minicluster/kylin_job_conf.xml b/examples/test_case_data/minicluster/kylin_job_conf.xml new file mode 100644 index 0000000..f62ce55 --- /dev/null +++ b/examples/test_case_data/minicluster/kylin_job_conf.xml @@ -0,0 +1,60 @@ + + + + + mapreduce.job.split.metainfo.maxsize + -1 + The maximum permissible size of the split metainfo file. + The JobTracker won't attempt to read split metainfo files bigger than + the configured value. No limits if set to -1. + + + + + + + mapreduce.job.max.split.locations + 2000 + No description + + + + dfs.replication + 2 + Block replication + + \ No newline at end of file diff --git a/examples/test_case_data/minicluster/mapred-site.xml b/examples/test_case_data/minicluster/mapred-site.xml new file mode 100644 index 0000000..7c65553 --- /dev/null +++ b/examples/test_case_data/minicluster/mapred-site.xml @@ -0,0 +1,8 @@ + + + + + mapred.job.tracker + localhost:9011 + + \ No newline at end of file diff --git a/job/pom.xml b/job/pom.xml index 90eb233..d63bcba 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -169,6 +169,22 @@ hadoop-hdfs provided + + org.apache.hbase + hbase-testing-util + ${hbase-hadoop2.version} + test + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + diff --git a/job/src/main/java/com/kylinolap/job/cmd/ShellCmd.java b/job/src/main/java/com/kylinolap/job/cmd/ShellCmd.java index d8d99c5..6ecb85c 100644 --- a/job/src/main/java/com/kylinolap/job/cmd/ShellCmd.java +++ b/job/src/main/java/com/kylinolap/job/cmd/ShellCmd.java @@ -62,7 +62,7 @@ protected ShellCmd(String executeCmd, ICommandOutput out, String host, String us this.output = out; this.remoteHost = host; this.remoteUser = user; - if (new File(password).exists()) { + if (password != null && new File(password).exists()) { this.identityPath = new File(password).getAbsolutePath(); this.remotePassword = null; } else { diff --git a/job/src/test/java/com/kylinolap/job/MiniClusterMetadataTestCase.java b/job/src/test/java/com/kylinolap/job/MiniClusterMetadataTestCase.java new file mode 100644 index 0000000..41eb1d6 --- /dev/null +++ b/job/src/test/java/com/kylinolap/job/MiniClusterMetadataTestCase.java @@ -0,0 +1,84 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.job; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MiniHBaseCluster; + +import com.kylinolap.common.KylinConfig; +import com.kylinolap.common.util.HBaseMetadataTestCase; + +/** + * @author shaoshi + */ +public class MiniClusterMetadataTestCase extends HBaseMetadataTestCase { + + private static final HBaseTestingUtility testUtil = new HBaseTestingUtility(); + private static MiniHBaseCluster hbCluster = null; + + @Override + public void createTestMetadata() { + staticCreateTestMetadata(MINICLUSTER_TEST_DATA); + //String connectionUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl(); + try { + hbCluster = testUtil.startMiniCluster(); + //testUtil.startMiniMapReduceCluster(); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + assert hbCluster != null; + + Configuration conf = hbCluster.getConf(); + String host = conf.get(HConstants.ZOOKEEPER_QUORUM); + String port = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT); + String parent = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT); + + // reduce rpc retry + conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000"); + conf.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5"); + conf.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000"); + + + String connectionUrl = "hbase:" + host + ":" + port + ":" + parent; + + KylinConfig.getInstanceFromEnv().setMetadataUrl(connectionUrl); + KylinConfig.getInstanceFromEnv().setStorageUrl(connectionUrl); + + } + + + + @Override + public void cleanupTestMetadata() { + + System.clearProperty(KylinConfig.KYLIN_CONF); + KylinConfig.destoryInstance(); + try { + testUtil.shutdownMiniMapReduceCluster(); + testUtil.shutdownMiniCluster(); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + +} diff --git a/job/src/test/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableTest.java b/job/src/test/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableTest.java index 2b44aed..f1c33cc 100644 --- a/job/src/test/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableTest.java +++ b/job/src/test/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableTest.java @@ -59,7 +59,7 @@ public void after() throws Exception { public void testGenCreateTableDDL() { String ddl = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, "/tmp", fakeJobUUID); System.out.println(ddl); - assertEquals(513, ddl.length()); + assertEquals(512, ddl.length()); } @Test @@ -74,7 +74,7 @@ public void testGenerateInsertSql() throws IOException { String sql = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, fakeJobUUID, new JobEngineConfig(KylinConfig.getInstanceFromEnv())); System.out.println(sql); - assertEquals(1239, sql.length()); + assertEquals(1127, sql.length()); } diff --git a/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java b/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java index 5c40eae..b1a8ba1 100644 --- a/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java +++ b/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java @@ -32,6 +32,7 @@ import org.junit.Test; import com.kylinolap.common.KylinConfig; +import com.kylinolap.common.util.AbstractKylinTestCase; import com.kylinolap.common.util.HBaseMetadataTestCase; import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.project.ProjectInstance; @@ -58,7 +59,7 @@ protected static void setupAll() throws SQLException { } private static void setUpEnv() { - HBaseMetadataTestCase.staticCreateTestMetadata(); + HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA); config = KylinConfig.getInstanceFromEnv(); } diff --git a/server/src/test/java/com/kylinolap/rest/service/ServiceTestBase.java b/server/src/test/java/com/kylinolap/rest/service/ServiceTestBase.java index 2d4229f..55b161b 100644 --- a/server/src/test/java/com/kylinolap/rest/service/ServiceTestBase.java +++ b/server/src/test/java/com/kylinolap/rest/service/ServiceTestBase.java @@ -29,6 +29,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import com.kylinolap.common.KylinConfig; +import com.kylinolap.common.util.AbstractKylinTestCase; import com.kylinolap.common.util.HBaseMetadataTestCase; import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.project.ProjectManager; @@ -45,7 +46,7 @@ @BeforeClass public static void setupResource() throws Exception { - staticCreateTestMetadata(); + staticCreateTestMetadata(SANDBOX_TEST_DATA); Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", "ROLE_ADMIN"); SecurityContextHolder.getContext().setAuthentication(authentication); diff --git a/storage/pom.xml b/storage/pom.xml index ecf3634..38e37c6 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -43,6 +43,17 @@ + org.apache.hive + hive-exec + ${hive.version} + + + org.apache.hive + hive-jdbc + provided + + + junit junit test diff --git a/storage/src/test/java/com/kylinolap/storage/minicluster/HiveJDBCClientTest.java b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveJDBCClientTest.java new file mode 100644 index 0000000..bbe76bc --- /dev/null +++ b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveJDBCClientTest.java @@ -0,0 +1,169 @@ +package com.kylinolap.storage.minicluster; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * A simple test case to check whether the Hive JDBC interface can fulfill Kylin's need; + * Before run it, you need startup the hive server on localhost: $HIVE_HOME/bin/hiveserver2 + * @author shaoshi + * + */ +public class HiveJDBCClientTest { + + private static String driverName = "org.apache.hive.jdbc.HiveDriver"; + + private static String tempFileName = "/tmp/a.txt"; + + //@Before + public void setup() { + + try { + Class.forName(driverName); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + System.exit(1); + } + + File testFile = new File(tempFileName); + + if (!testFile.exists()) { + + FileWriter writer; + try { + writer = new FileWriter(testFile); + writer.write("1 a\n"); + writer.write("2 b\n"); + + writer.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + } + + //@After + public void tearDown() { + + } + + protected Connection getHiveConnection() throws SQLException { + + return DriverManager.getConnection("jdbc:hive2://localhost:10000/default", "hive", ""); + } + + //@Test + public void testConnectToHive() throws SQLException { + + //replace "hive" here with the name of the user the queries should run as + Connection con = getHiveConnection(); + + Statement stmt = con.createStatement(); + String tableName = "testHiveDriverTable"; + stmt.execute("drop table if exists " + tableName); + stmt.execute("create table " + tableName + " (key int, value string) row format delimited fields terminated by ' '"); + // show tables + String sql = "show tables '" + tableName + "'"; + System.out.println("Running: " + sql); + ResultSet res = stmt.executeQuery(sql); + if (res.next()) { + System.out.println(res.getString(1)); + } + res.close(); + + // describe table + sql = "describe " + tableName; + System.out.println("Running: " + sql); + res = stmt.executeQuery(sql); + while (res.next()) { + System.out.println(res.getString(1) + "\t" + res.getString(2)); + } + res.close(); + + // load data into table + // NOTE: filepath has to be local to the hive server + // NOTE: /tmp/a.txt is a ctrl-A separated file with two fields per line + String filepath = tempFileName; + sql = "load data local inpath '" + filepath + "' into table " + tableName; + System.out.println("Running: " + sql); + stmt.execute(sql); + + // select * query + sql = "select * from " + tableName; + System.out.println("Running: " + sql); + res = stmt.executeQuery(sql); + while (res.next()) { + System.out.println(String.valueOf(res.getInt(1)) + "\t" + res.getString(2)); + } + res.close(); + + // regular hive query + sql = "select count(1) from " + tableName; + System.out.println("Running: " + sql); + res = stmt.executeQuery(sql); + while (res.next()) { + System.out.println(res.getInt(1)); + } + res.close(); + } + + //@Test + public void testShowExtendedTable() throws SQLException { + Connection con = getHiveConnection(); + + String tableName = "testHiveDriverTable"; + String sql = "show table extended like " + tableName + ""; + + ResultSet res = executeSQL(con, sql); + while (res.next()) { + System.out.println("--- " + res.getString("tab_name")); + } + + sql = "describe extended " + tableName + ""; + res = executeSQL(con, sql); + while (res.next()) { + System.out.println("---" + res.getString(1) + " | " + res.getString(2)); + } + + } + + protected ResultSet executeSQL(Connection con, String sql) throws SQLException { + + Statement stmt = con.createStatement(); + System.out.println("Running: " + sql); + stmt.execute(sql); + + return stmt.getResultSet(); + } + + public void runTests() throws SQLException { + try { + setup(); + testConnectToHive(); + testShowExtendedTable(); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + tearDown(); + } + + } + + public static void main(String[] args) throws SQLException { + HiveJDBCClientTest test = new HiveJDBCClientTest(); + test.runTests(); + } + +} diff --git a/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest.java b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest.java new file mode 100644 index 0000000..52be0f9 --- /dev/null +++ b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest.java @@ -0,0 +1,117 @@ +package com.kylinolap.storage.minicluster; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.service.HiveInterface; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MiniMRCluster; +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * This is a test case to verify whether the query can be executed on Hive minicluster; + * You need set $HADOOP_HOME environment parameter before run it; + * @author shaoshi + * + */ +public class HiveMiniClusterTest extends HiveJDBCClientTest { + public static final File HIVE_BASE_DIR = new File("target/hive"); + public static final File HIVE_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/scratchdir"); + public static final File HIVE_LOCAL_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/localscratchdir"); + public static final File HIVE_METADB_DIR = new File(HIVE_BASE_DIR + "/metastoredb"); + public static final File HIVE_LOGS_DIR = new File(HIVE_BASE_DIR + "/logs"); + public static final File HIVE_TMP_DIR = new File(HIVE_BASE_DIR + "/tmp"); + public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR + "/warehouse"); + public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR + "/testdata"); + public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR + "/hadooptmp"); + protected HiveInterface client; + + protected MiniDFSCluster miniDFS; + protected MiniMRCluster miniMR; + + //@Before + public void setup() { + super.setup(); + startHiveMiniCluster(); + } + + protected void startHiveMiniCluster() { + //Create and configure location for hive to dump junk in target folder + try { + FileUtils.forceMkdir(HIVE_BASE_DIR); + FileUtils.forceMkdir(HIVE_SCRATCH_DIR); + FileUtils.forceMkdir(HIVE_LOCAL_SCRATCH_DIR); + FileUtils.forceMkdir(HIVE_LOGS_DIR); + FileUtils.forceMkdir(HIVE_TMP_DIR); + FileUtils.forceMkdir(HIVE_WAREHOUSE_DIR); + FileUtils.forceMkdir(HIVE_HADOOP_TMP_DIR); + FileUtils.forceMkdir(HIVE_TESTDATA_DIR); + } catch (IOException e1) { + e1.printStackTrace(); + System.exit(1); + } + + System.setProperty("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=" + HIVE_METADB_DIR.getAbsolutePath() + ";create=true"); + System.setProperty("hive.metastore.warehouse.dir", HIVE_WAREHOUSE_DIR.getAbsolutePath()); + System.setProperty("hive.exec.scratchdir", HIVE_SCRATCH_DIR.getAbsolutePath()); + System.setProperty("hive.exec.local.scratchdir", HIVE_LOCAL_SCRATCH_DIR.getAbsolutePath()); + System.setProperty("hive.metastore.metadb.dir", HIVE_METADB_DIR.getAbsolutePath()); + System.setProperty("test.log.dir", HIVE_LOGS_DIR.getAbsolutePath()); + System.setProperty("hive.querylog.location", HIVE_TMP_DIR.getAbsolutePath()); + System.setProperty("hadoop.tmp.dir", HIVE_HADOOP_TMP_DIR.getAbsolutePath()); + System.setProperty("derby.stream.error.file", HIVE_BASE_DIR.getAbsolutePath() + "/derby.log"); + + // custom properties + System.setProperty("hive.server2.long.polling.timeout", "5000"); + + HiveConf conf = new HiveConf(); + + /* Build MiniDFSCluster */ + try { + miniDFS = new MiniDFSCluster.Builder(conf).build(); + + /* Build MiniMR Cluster */ + int numTaskTrackers = 1; + int numTaskTrackerDirectories = 1; + String[] racks = null; + String[] hosts = null; + miniMR = new MiniMRCluster(numTaskTrackers, miniDFS.getFileSystem().getUri().toString(), numTaskTrackerDirectories, racks, hosts, new JobConf(conf)); + + System.setProperty("mapred.job.tracker", miniMR.createJobConf(new JobConf(conf)).get("mapred.job.tracker")); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + } + + //@After + public void tearDown() { + if (miniMR != null) + miniMR.shutdown(); + + if (miniDFS != null) + miniDFS.shutdown(); + + super.tearDown(); + } + + protected Connection getHiveConnection() throws SQLException { + return DriverManager.getConnection("jdbc:hive2:///", "", ""); + } + + public static void main(String[] args) throws SQLException { + HiveMiniClusterTest test = new HiveMiniClusterTest(); + test.runTests(); + } +} diff --git a/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest2.java b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest2.java new file mode 100644 index 0000000..a411571 --- /dev/null +++ b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest2.java @@ -0,0 +1,126 @@ +package com.kylinolap.storage.minicluster; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.service.HiveInterface; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MiniMRCluster; + +/** + * This is a similar test with HiveMiniClusterTest.java ; except need set HADOOP_HOME, it also need give bigger JVM heap size, like + * "-Xmx2048m -Xms1024m -XX:PermSize=256M -XX:MaxPermSize=512m" + * + * @deprecated Please see HiveMiniClusterTest3.java + * @author shaoshi + * + */ +public class HiveMiniClusterTest2 extends HiveJDBCClientTest { + public static final File HIVE_BASE_DIR = new File("target/hive"); + public static final File HIVE_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/scratchdir"); + public static final File HIVE_LOCAL_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/localscratchdir"); + public static final File HIVE_METADB_DIR = new File(HIVE_BASE_DIR + "/metastoredb"); + public static final File HIVE_LOGS_DIR = new File(HIVE_BASE_DIR + "/logs"); + public static final File HIVE_TMP_DIR = new File(HIVE_BASE_DIR + "/tmp"); + public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR + "/warehouse"); + public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR + "/testdata"); + public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR + "/hadooptmp"); + protected HiveInterface client; + + private static final HBaseTestingUtility testUtil = new HBaseTestingUtility(); + + //@Before + public void setup() { + super.setup(); + startHiveMiniCluster(); + } + + protected void startHiveMiniCluster() { + //Create and configure location for hive to dump junk in target folder + try { + FileUtils.forceMkdir(HIVE_BASE_DIR); + FileUtils.forceMkdir(HIVE_SCRATCH_DIR); + FileUtils.forceMkdir(HIVE_LOCAL_SCRATCH_DIR); + FileUtils.forceMkdir(HIVE_LOGS_DIR); + FileUtils.forceMkdir(HIVE_TMP_DIR); + FileUtils.forceMkdir(HIVE_WAREHOUSE_DIR); + FileUtils.forceMkdir(HIVE_HADOOP_TMP_DIR); + FileUtils.forceMkdir(HIVE_TESTDATA_DIR); + } catch (IOException e1) { + e1.printStackTrace(); + System.exit(1); + } + + System.setProperty("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=" + HIVE_METADB_DIR.getAbsolutePath() + ";create=true"); + System.setProperty("hive.metastore.warehouse.dir", HIVE_WAREHOUSE_DIR.getAbsolutePath()); + System.setProperty("hive.exec.scratchdir", HIVE_SCRATCH_DIR.getAbsolutePath()); + System.setProperty("hive.exec.local.scratchdir", HIVE_LOCAL_SCRATCH_DIR.getAbsolutePath()); + System.setProperty("hive.metastore.metadb.dir", HIVE_METADB_DIR.getAbsolutePath()); + System.setProperty("test.log.dir", HIVE_LOGS_DIR.getAbsolutePath()); + System.setProperty("hive.querylog.location", HIVE_TMP_DIR.getAbsolutePath()); + System.setProperty("hadoop.tmp.dir", HIVE_HADOOP_TMP_DIR.getAbsolutePath()); + System.setProperty("derby.stream.error.file", HIVE_BASE_DIR.getAbsolutePath() + "/derby.log"); + + // custom properties + System.setProperty("hive.server2.long.polling.timeout", "5000"); + + HiveConf conf = new HiveConf(); + + try { + MiniHBaseCluster miniCluster = testUtil.startMiniCluster(1); + //MiniDFSCluster dfsCluster = testUtil.getDFSCluster(); + MiniMRCluster miniMR = testUtil.startMiniMapReduceCluster(); + + Configuration config = miniCluster.getConf(); + // System.out.println("-----------------------------"); + // System.out.println(config.get("yarn.resourcemanager.scheduler.address")); + // System.out.println(config.get("mapreduce.jobtracker.address")); + // System.out.println(config.get("yarn.resourcemanager.address")); + // System.out.println(config.get("mapreduce.jobhistory.address")); + // System.out.println(config.get("yarn.resourcemanager.webapp.address")); + // System.out.println("-----------------------------"); + // + //System.setProperty("mapred.job.tracker", miniMR.createJobConf(new JobConf(config)).get("mapred.job.tracker")); + String jobTracker = miniMR.createJobConf(new JobConf(config)).get("mapreduce.jobtracker.address"); + System.setProperty("mapred.job.tracker", jobTracker); + System.setProperty("mapreduce.jobtracker.address", jobTracker); + System.setProperty("mapred.job.tracker", miniMR.createJobConf(new JobConf(conf)).get("mapred.job.tracker")); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + + //@After + public void tearDown() { + + try { + //testUtil.shutdownMiniMapReduceCluster(); + testUtil.shutdownMiniCluster(); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + super.tearDown(); + } + + protected Connection getHiveConnection() throws SQLException { + return DriverManager.getConnection("jdbc:hive2:///", "", ""); + } + + public static void main(String[] args) throws SQLException { + HiveMiniClusterTest2 test = new HiveMiniClusterTest2(); + test.runTests(); + } + +} diff --git a/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest3.java b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest3.java new file mode 100644 index 0000000..43f8f01 --- /dev/null +++ b/storage/src/test/java/com/kylinolap/storage/minicluster/HiveMiniClusterTest3.java @@ -0,0 +1,205 @@ +package com.kylinolap.storage.minicluster; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.service.HiveInterface; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MiniMRCluster; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +public class HiveMiniClusterTest3 extends HiveJDBCClientTest { + public static final File HIVE_BASE_DIR = new File("target/hive"); + public static final File HIVE_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/scratchdir"); + public static final File HIVE_LOCAL_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/localscratchdir"); + public static final File HIVE_METADB_DIR = new File(HIVE_BASE_DIR + "/metastoredb"); + public static final File HIVE_LOGS_DIR = new File(HIVE_BASE_DIR + "/logs"); + public static final File HIVE_TMP_DIR = new File(HIVE_BASE_DIR + "/tmp"); + public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR + "/warehouse"); + public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR + "/testdata"); + public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR + "/hadooptmp"); + protected HiveInterface client; + + private static final HBaseTestingUtility testUtil = new HBaseTestingUtility(); + + protected static MiniDFSCluster dfsCluster = null; + protected static MiniMRCluster mrCluster = null; + protected static JobConf conf = null; + private static final int NAMENODE_PORT = 9010; + private static final int JOBTRACKER_PORT = 9011; + protected static String LOG_DIR = "/tmp/logs"; + + private static class ConfigurableMiniMRCluster extends MiniMRCluster { + private static Properties config; + + public static void setConfiguration(Properties props) { + config = props; + } + + public ConfigurableMiniMRCluster(int numTaskTrackers, String namenode, + int numDir, JobConf conf) + throws Exception { + super(JOBTRACKER_PORT,0, numTaskTrackers, namenode, numDir, null, null, null, conf); + } + + public JobConf createJobConf() { + JobConf conf = super.createJobConf(); + if (config != null) { + for (Map.Entry entry : config.entrySet()) { + conf.set((String) entry.getKey(), (String) entry.getValue()); + } + } + return conf; + } + } + + //@Before + public void setup() { + try { + setUpBeforeClass(); + } catch (Exception e) { + e.printStackTrace(); + } + super.setup(); + startHiveMiniCluster(); + } + + protected void startHiveMiniCluster() { + //Create and configure location for hive to dump junk in target folder + try { + FileUtils.forceMkdir(HIVE_BASE_DIR); + FileUtils.forceMkdir(HIVE_SCRATCH_DIR); + FileUtils.forceMkdir(HIVE_LOCAL_SCRATCH_DIR); + FileUtils.forceMkdir(HIVE_LOGS_DIR); + FileUtils.forceMkdir(HIVE_TMP_DIR); + FileUtils.forceMkdir(HIVE_WAREHOUSE_DIR); + FileUtils.forceMkdir(HIVE_HADOOP_TMP_DIR); + FileUtils.forceMkdir(HIVE_TESTDATA_DIR); + } catch (IOException e1) { + e1.printStackTrace(); + System.exit(1); + } + + System.setProperty("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=" + HIVE_METADB_DIR.getAbsolutePath() + ";create=true"); + System.setProperty("hive.metastore.warehouse.dir", HIVE_WAREHOUSE_DIR.getAbsolutePath()); + System.setProperty("hive.exec.scratchdir", HIVE_SCRATCH_DIR.getAbsolutePath()); + System.setProperty("hive.exec.local.scratchdir", HIVE_LOCAL_SCRATCH_DIR.getAbsolutePath()); + System.setProperty("hive.metastore.metadb.dir", HIVE_METADB_DIR.getAbsolutePath()); + System.setProperty("test.log.dir", HIVE_LOGS_DIR.getAbsolutePath()); + System.setProperty("hive.querylog.location", HIVE_TMP_DIR.getAbsolutePath()); + System.setProperty("hadoop.tmp.dir", HIVE_HADOOP_TMP_DIR.getAbsolutePath()); + System.setProperty("derby.stream.error.file", HIVE_BASE_DIR.getAbsolutePath() + "/derby.log"); + System.setProperty("mapred.job.tracker", "localhost:9011"); + + // custom properties + System.setProperty("hive.server2.long.polling.timeout", "5000"); + + } + + + //@BeforeClass + public static void setUpBeforeClass() throws Exception { + // make sure the log dir exists + File logPath = new File(LOG_DIR); + if (!logPath.exists()){ + logPath.mkdirs(); + } + // configure and start the cluster + System.setProperty("hadoop.log.dir", LOG_DIR); + System.setProperty("javax.xml.parsers.SAXParserFactory", + "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"); + System.setProperty("javax.xml.parsers.DocumentBuilderFactory", + "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"); + Properties props = new Properties(); + props.setProperty("dfs.datanode.data.dir.perm", "775"); + conf = new JobConf(); + String hadoopConfDir = "test" + File.separator + "resources" + File.separator + "hadoop" + File.separator + "conf"; + File hadoopConfFolder = new File(hadoopConfDir); + if(!hadoopConfFolder.exists()) { + System.err.println("Couldn't find " + hadoopConfDir + ", exit..."); + System.exit(1); + } + + String coreSitePath = hadoopConfDir + File.separator + "core-site.xml"; + conf.addResource(new Path(coreSitePath)); + String hdfsSitePath = hadoopConfDir + File.separator + "hdfs-site.xml"; + conf.addResource(new Path(hdfsSitePath)); + String mrSitePath = hadoopConfDir + File.separator + "mapred-site.xml"; + conf.addResource(new Path(mrSitePath)); + startCluster(true, conf, props); + + + } + + public void tearDown() { + try { + tearDownAfterClass(); + } catch (Exception e) { + e.printStackTrace(); + } + super.tearDown(); + } + + //@AfterClass + public static void tearDownAfterClass() throws Exception { + stopCluster(); + // clean up the hdfs files created by mini cluster + String baseTempDir = "build" + File.separator + "test" + File.separator; + String dfsDir = baseTempDir + "data"; + FileUtils.deleteDirectory(new File(dfsDir)); + String mrDir = baseTempDir + "mapred"; + FileUtils.deleteDirectory(new File(mrDir)); + FileUtils.cleanDirectory(new File(LOG_DIR)); + } + + protected static synchronized void startCluster(boolean reformatDFS, JobConf conf ,Properties props) + throws Exception { + if (dfsCluster == null) { + + if (props != null) { + for (Map.Entry entry : props.entrySet()) { + conf.set((String) entry.getKey(), (String) entry.getValue()); + } + } + dfsCluster = new MiniDFSCluster(NAMENODE_PORT, conf, 2, reformatDFS, true, null, null); + ConfigurableMiniMRCluster.setConfiguration(props); + mrCluster = new ConfigurableMiniMRCluster(2, dfsCluster.getFileSystem().getName(), + 1, conf); + } + } + + protected static void stopCluster() throws Exception { + if (mrCluster != null) { + mrCluster.shutdown(); + mrCluster = null; + } + if (dfsCluster != null) { + dfsCluster.shutdown(); + dfsCluster = null; + } + } + + + protected Connection getHiveConnection() throws SQLException { + return DriverManager.getConnection("jdbc:hive2:///", "hive", ""); + } + + public static void main(String[] args) throws SQLException { + HiveMiniClusterTest test = new HiveMiniClusterTest(); + test.runTests(); + } + +}