diff --git a/data/conf/hbase/hive-site.xml b/data/conf/hbase/hive-site.xml new file mode 100644 index 0000000..2cde40f --- /dev/null +++ b/data/conf/hbase/hive-site.xml @@ -0,0 +1,263 @@ + + + + + + + + hive.in.test + true + Internal marker for test. Used for masking env-dependent values + + + + + + + + + + + hadoop.tmp.dir + ${test.tmp.dir}/hadoop-tmp + A base for other temporary directories. + + + + + + hive.exec.scratchdir + ${test.tmp.dir}/scratchdir + Scratch space for Hive jobs + + + + hive.exec.local.scratchdir + ${test.tmp.dir}/localscratchdir/ + Local scratch space for Hive jobs + + + + javax.jdo.option.ConnectionURL + jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true + + + + hive.stats.dbconnectionstring + jdbc:derby:;databaseName=${test.tmp.dir}/TempStatsStore;create=true + + + + + javax.jdo.option.ConnectionDriverName + org.apache.derby.jdbc.EmbeddedDriver + + + + javax.jdo.option.ConnectionUserName + APP + + + + javax.jdo.option.ConnectionPassword + mine + + + + + hive.metastore.warehouse.dir + ${test.warehouse.dir} + + + + + hive.metastore.metadb.dir + file://${test.tmp.dir}/metadb/ + + Required by metastore server or if the uris argument below is not supplied + + + + + test.log.dir + ${test.tmp.dir}/log/ + + + + + test.data.files + ${hive.root}/data/files + + + + + test.data.scripts + ${hive.root}/data/scripts + + + + + hive.jar.path + ${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar + + + + + hive.querylog.location + ${test.tmp.dir}/tmp + Location of the structured hive logs + + + + hive.exec.pre.hooks + org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables + Pre Execute Hook for Tests + + + + hive.exec.post.hooks + org.apache.hadoop.hive.ql.hooks.PostExecutePrinter + Post Execute Hook for Tests + + + + hive.support.concurrency + false + Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks. + + + + hive.unlock.numretries + 2 + The number of times you want to retry to do one unlock + + + + hive.lock.sleep.between.retries + 2 + The sleep time (in seconds) between various retries + + + + + fs.pfile.impl + org.apache.hadoop.fs.ProxyLocalFileSystem + A proxy for local file system used for cross file system testing + + + + hive.exec.mode.local.auto + false + + Let hive determine whether to run in local mode automatically + Disabling this for tests so that minimr is not affected + + + + + hive.auto.convert.join + false + Whether Hive enable the optimization about converting common join into mapjoin based on the input file size + + + + hive.ignore.mapjoin.hint + false + Whether Hive ignores the mapjoin hint + + + + hive.input.format + org.apache.hadoop.hive.ql.io.CombineHiveInputFormat + The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. + + + + hive.default.rcfile.serde + org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + The default SerDe hive will use for the rcfile format + + + + hive.stats.dbclass + jdbc:derby + The storage for temporary stats generated by tasks. Currently, jdbc, hbase and counter types are supported + + + + hive.stats.key.prefix.reserve.length + 0 + + + + hive.conf.restricted.list + dummy.config.value + Using dummy config value above because you cannot override config with empty value + + + + hive.exec.submit.local.task.via.child + false + + + + + hive.dummyparam.test.server.specific.config.override + from.hive-site.xml + Using dummy param to test server specific configuration + + + + hive.dummyparam.test.server.specific.config.hivesite + from.hive-site.xml + Using dummy param to test server specific configuration + + + + hive.ql.log.PerfLogger.level + WARN,DRFA + Used to change the perflogger level + + + + hive.fetch.task.conversion + minimal + + + + hive.users.in.admin.role + hive_admin_user + + + + hive.metastore.fastpath + true + + + + hive.metastore.rawstore.impl + org.apache.hadoop.hive.metastore.hbase.HBaseStore + + + diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index e195bee..a48772e 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -532,6 +532,25 @@ + + org.apache.hbase hbase-server ${hbase.hadoop2.version} + test-jar + + + org.apache.hbase + hbase-common + ${hbase.hadoop2.version} + test-jar + + + org.apache.hbase + hbase-server + ${hbase.hadoop2.version} org.apache.hbase diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 39d5d9e..efdebd7 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql; +import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_COMMENT; import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME; import java.io.BufferedInputStream; @@ -65,6 +66,11 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hive.cli.CliDriver; import org.apache.hadoop.hive.cli.CliSessionState; @@ -75,7 +81,10 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.hbase.HBaseReadWrite; +import org.apache.hadoop.hive.metastore.hbase.TephraHBaseConnection; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -162,6 +171,8 @@ public interface SuiteAddTestFunctor { public void addTestToSuite(TestSuite suite, Object setup, String tName); } + private HBaseTestingUtility utility; + private boolean snapshotTaken = false; static { for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) { @@ -277,10 +288,6 @@ public void initConf() throws Exception { conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, true); } - // Plug verifying metastore in for testing. - conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, - "org.apache.hadoop.hive.metastore.VerifyingObjectStore"); - if (mr != null) { assert dfs != null; @@ -342,6 +349,51 @@ private String getKeyProviderURI() { return "jceks://file" + new Path(keyDir, "test.jks").toUri(); } + private void rebuildHBase() throws Exception { + HBaseAdmin admin = utility.getHBaseAdmin(); + if (!snapshotTaken) { + for (String tableName : HBaseReadWrite.tableNames) { + List families = HBaseReadWrite.columnFamilies.get(tableName); + HTableDescriptor desc = new HTableDescriptor( + TableName.valueOf(tableName)); + for (byte[] family : families) { + HColumnDescriptor columnDesc = new HColumnDescriptor(family); + desc.addFamily(columnDesc); + } + try { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } catch (IOException e) { + System.out.println(e.getMessage()); + } + admin.createTable(desc); + } + } else { + for (String tableName : HBaseReadWrite.tableNames) { + admin.disableTable(tableName); + admin.restoreSnapshot("snapshot_" + tableName); + admin.enableTable(tableName); + } + try { + db.createDatabase(new org.apache.hadoop.hive.metastore.api.Database( + DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT, new Warehouse(conf) + .getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null)); + } catch (Exception e) { + // Ignore if default database already exist + } + SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME); + } + admin.close(); + } + + private void startMiniHBaseCluster() throws Exception { + utility = new HBaseTestingUtility(); + utility.startMiniCluster(); + conf = new HiveConf(utility.getConfiguration(), Driver.class); + rebuildHBase(); + HBaseReadWrite.getInstance(conf); + } + public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript) throws Exception { @@ -351,6 +403,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml")); System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation()); } + startMiniHBaseCluster(); conf = new HiveConf(Driver.class); this.hadoopVer = getHadoopMainVersion(hadoopVer); qMap = new TreeMap(); @@ -444,6 +497,7 @@ public void shutdown() throws Exception { sparkSession = null; } } + utility.shutdownMiniCluster(); if (mr != null) { mr.shutdown(); mr = null; @@ -731,6 +785,8 @@ public void clearTestSideEffects() throws Exception { return; } + rebuildHBase(); + clearTablesCreatedDuringTests(); clearKeysCreatedInTests(); @@ -820,6 +876,12 @@ public void createSources() throws Exception { cliDriver.processLine(initCommands); conf.setBoolean("hive.test.init.phase", false); + + HBaseAdmin admin = utility.getHBaseAdmin(); + for (String tableName : HBaseReadWrite.tableNames) { + admin.snapshot("snapshot_" + tableName, tableName); + } + snapshotTaken = true; } public void init() throws Exception { @@ -1571,7 +1633,7 @@ private static int executeDiffCommand(String inFileName, // close it first. SessionState ss = SessionState.get(); if (ss != null && ss.out != null && ss.out != System.out) { - ss.out.close(); + ss.out.close(); } String inSorted = inFileName + SORT_SUFFIX; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java index ae73feb..08dd7e3 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java @@ -71,7 +71,7 @@ /** * Class to manage storing object in and reading them from HBase. */ -class HBaseReadWrite { +public class HBaseReadWrite { @VisibleForTesting final static String AGGR_STATS_TABLE = "HBMS_AGGR_STATS"; @VisibleForTesting final static String DB_TABLE = "HBMS_DBS"; @@ -90,10 +90,9 @@ /** * List of tables in HBase */ - final static String[] tableNames = { AGGR_STATS_TABLE, DB_TABLE, FUNC_TABLE, GLOBAL_PRIVS_TABLE, - PART_TABLE, USER_TO_ROLE_TABLE, ROLE_TABLE, SD_TABLE, - SECURITY_TABLE, SEQUENCES_TABLE, TABLE_TABLE}; - final static Map> columnFamilies = + final static public String[] tableNames = { DB_TABLE, FUNC_TABLE, GLOBAL_PRIVS_TABLE, PART_TABLE, + USER_TO_ROLE_TABLE, ROLE_TABLE, SD_TABLE, TABLE_TABLE }; + final static public Map> columnFamilies = new HashMap> (tableNames.length); static { @@ -178,7 +177,7 @@ protected HBaseReadWrite initialValue() { * @param configuration Configuration object * @return thread's instance of HBaseReadWrite */ - static HBaseReadWrite getInstance(Configuration configuration) { + public static HBaseReadWrite getInstance(Configuration configuration) { staticConf = configuration; return self.get(); }