diff --git a/data/conf/hbase/hive-site.xml b/data/conf/hbase/hive-site.xml deleted file mode 100644 index 2cde40f..0000000 --- a/data/conf/hbase/hive-site.xml +++ /dev/null @@ -1,263 +0,0 @@ - - - - - - - - hive.in.test - true - Internal marker for test. Used for masking env-dependent values - - - - - - - - - - - hadoop.tmp.dir - ${test.tmp.dir}/hadoop-tmp - A base for other temporary directories. - - - - - - hive.exec.scratchdir - ${test.tmp.dir}/scratchdir - Scratch space for Hive jobs - - - - hive.exec.local.scratchdir - ${test.tmp.dir}/localscratchdir/ - Local scratch space for Hive jobs - - - - javax.jdo.option.ConnectionURL - jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true - - - - hive.stats.dbconnectionstring - jdbc:derby:;databaseName=${test.tmp.dir}/TempStatsStore;create=true - - - - - javax.jdo.option.ConnectionDriverName - org.apache.derby.jdbc.EmbeddedDriver - - - - javax.jdo.option.ConnectionUserName - APP - - - - javax.jdo.option.ConnectionPassword - mine - - - - - hive.metastore.warehouse.dir - ${test.warehouse.dir} - - - - - hive.metastore.metadb.dir - file://${test.tmp.dir}/metadb/ - - Required by metastore server or if the uris argument below is not supplied - - - - - test.log.dir - ${test.tmp.dir}/log/ - - - - - test.data.files - ${hive.root}/data/files - - - - - test.data.scripts - ${hive.root}/data/scripts - - - - - hive.jar.path - ${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar - - - - - hive.querylog.location - ${test.tmp.dir}/tmp - Location of the structured hive logs - - - - hive.exec.pre.hooks - org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables - Pre Execute Hook for Tests - - - - hive.exec.post.hooks - org.apache.hadoop.hive.ql.hooks.PostExecutePrinter - Post Execute Hook for Tests - - - - hive.support.concurrency - false - Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks. - - - - hive.unlock.numretries - 2 - The number of times you want to retry to do one unlock - - - - hive.lock.sleep.between.retries - 2 - The sleep time (in seconds) between various retries - - - - - fs.pfile.impl - org.apache.hadoop.fs.ProxyLocalFileSystem - A proxy for local file system used for cross file system testing - - - - hive.exec.mode.local.auto - false - - Let hive determine whether to run in local mode automatically - Disabling this for tests so that minimr is not affected - - - - - hive.auto.convert.join - false - Whether Hive enable the optimization about converting common join into mapjoin based on the input file size - - - - hive.ignore.mapjoin.hint - false - Whether Hive ignores the mapjoin hint - - - - hive.input.format - org.apache.hadoop.hive.ql.io.CombineHiveInputFormat - The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. - - - - hive.default.rcfile.serde - org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - The default SerDe hive will use for the rcfile format - - - - hive.stats.dbclass - jdbc:derby - The storage for temporary stats generated by tasks. Currently, jdbc, hbase and counter types are supported - - - - hive.stats.key.prefix.reserve.length - 0 - - - - hive.conf.restricted.list - dummy.config.value - Using dummy config value above because you cannot override config with empty value - - - - hive.exec.submit.local.task.via.child - false - - - - - hive.dummyparam.test.server.specific.config.override - from.hive-site.xml - Using dummy param to test server specific configuration - - - - hive.dummyparam.test.server.specific.config.hivesite - from.hive-site.xml - Using dummy param to test server specific configuration - - - - hive.ql.log.PerfLogger.level - WARN,DRFA - Used to change the perflogger level - - - - hive.fetch.task.conversion - minimal - - - - hive.users.in.admin.role - hive_admin_user - - - - hive.metastore.fastpath - true - - - - hive.metastore.rawstore.impl - org.apache.hadoop.hive.metastore.hbase.HBaseStore - - - diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java index 1ac10f0..267152e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseImport.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.RawStore; +import org.apache.hadoop.hive.metastore.TestObjectStore; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Function; @@ -57,12 +58,13 @@ */ public class TestHBaseImport extends HBaseIntegrationTests { - private static final Log LOG = LogFactory.getLog(TestHBaseStoreIntegration.class.getName()); + private static final Log LOG = LogFactory.getLog(TestHBaseImport.class.getName()); private static final String[] tableNames = new String[] {"allnonparttable", "allparttable"}; private static final String[] partVals = new String[] {"na", "emea", "latam", "apac"}; private static final String[] funcNames = new String[] {"allfunc1", "allfunc2"}; + private static final List masterKeySeqs = new ArrayList(); @Rule public ExpectedException thrown = ExpectedException.none(); @@ -73,6 +75,13 @@ public static void startup() throws Exception { @AfterClass public static void shutdown() throws Exception { + RawStore rdbms; + rdbms = new ObjectStore(); + rdbms.setConf(conf); + TestObjectStore.dropAllStoreObjects(rdbms); + for (int seq : masterKeySeqs) { + rdbms.removeMasterKey(seq); + } HBaseIntegrationTests.shutdownMiniCluster(); } @@ -316,7 +325,6 @@ public void importOneTableNonPartitioned() throws Exception { Assert.assertEquals(baseNumToks, store.getAllTokenIdentifiers().size()); String[] hbaseKeys = store.getMasterKeys(); Assert.assertEquals(baseNumKeys, hbaseKeys.length); - } @Test @@ -502,7 +510,7 @@ private void setupObjectStore(RawStore rdbms, String[] roles, String[] dbNames, } for (int i = 0; i < tokenIds.length; i++) rdbms.addToken(tokenIds[i], tokens[i]); for (int i = 0; i < masterKeys.length; i++) { - rdbms.addMasterKey(masterKeys[i]); + masterKeySeqs.add(rdbms.addMasterKey(masterKeys[i])); } } diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index 664068b..122e3f6 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -533,25 +533,6 @@ - - co.cask.tephra tephra-core ${tephra.version} + + + org.ow2.asm + asm-all + + co.cask.tephra diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java index a4f9f6c..d05d833 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -69,21 +70,15 @@ public boolean filterPartitionsByExpr(List partColumnNames, } @Before - public void setUp() { + public void setUp() throws Exception { HiveConf conf = new HiveConf(); conf.setVar(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, MockPartitionExpressionProxy.class.getName()); objectStore = new ObjectStore(); objectStore.setConf(conf); - - Deadline.registerIfNot(100000); try { - objectStore.dropDatabase(DB1); - } catch (Exception e) { - } - try { - objectStore.dropDatabase(DB2); - } catch (Exception e) { + dropAllStoreObjects(objectStore); + } catch (NoSuchObjectException e) { } } @@ -227,4 +222,30 @@ public void testRoleOps() throws InvalidObjectException, MetaException, NoSuchOb objectStore.revokeRole(role1, USER1, PrincipalType.USER, false); objectStore.removeRole(ROLE1); } + + public static void dropAllStoreObjects(RawStore store) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { + Deadline.registerIfNot(100000); + List funcs = store.getAllFunctions(); + for (Function func : funcs) { + store.dropFunction(func.getDbName(), func.getFunctionName()); + } + List dbs = store.getAllDatabases(); + for (int i = 0; i < dbs.size(); i++) { + String db = dbs.get(i); + List tbls = store.getAllTables(db); + for (String tbl : tbls) { + Deadline.startTimer("getPartition"); + List parts = store.getPartitions(db, tbl, 100); + for (Partition part : parts) { + store.dropPartition(db, tbl, part.getValues()); + } + store.dropTable(db, tbl); + } + store.dropDatabase(db); + } + List roles = store.listRoleNames(); + for (String role : roles) { + store.removeRole(role); + } + } } diff --git a/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out index 216a79c..1f6339a 100644 --- a/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out +++ b/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out @@ -1755,13 +1755,15 @@ Bucket Columns: [] Sort Columns: [] Storage Desc Params: serialization.format 1 -PREHOOK: query: select * from over1k_part2_orc +PREHOOK: query: -- SORT_BEFORE_DIFF +select * from over1k_part2_orc PREHOOK: type: QUERY PREHOOK: Input: default@over1k_part2_orc PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=27 PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### -POSTHOOK: query: select * from over1k_part2_orc +POSTHOOK: query: -- SORT_BEFORE_DIFF +select * from over1k_part2_orc POSTHOOK: type: QUERY POSTHOOK: Input: default@over1k_part2_orc POSTHOOK: Input: default@over1k_part2_orc@ds=foo/t=27 @@ -1900,13 +1902,15 @@ Bucket Columns: [] Sort Columns: [] Storage Desc Params: serialization.format 1 -PREHOOK: query: select * from over1k_part2_orc +PREHOOK: query: -- SORT_BEFORE_DIFF +select * from over1k_part2_orc PREHOOK: type: QUERY PREHOOK: Input: default@over1k_part2_orc PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=27 PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### -POSTHOOK: query: select * from over1k_part2_orc +POSTHOOK: query: -- SORT_BEFORE_DIFF +select * from over1k_part2_orc POSTHOOK: type: QUERY POSTHOOK: Input: default@over1k_part2_orc POSTHOOK: Input: default@over1k_part2_orc@ds=foo/t=27 diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out index 41049bd..ebf4461 100644 --- a/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out @@ -1655,13 +1655,15 @@ Bucket Columns: [] Sort Columns: [] Storage Desc Params: serialization.format 1 -PREHOOK: query: select * from over1k_part2 +PREHOOK: query: -- SORT_BEFORE_DIFF +select * from over1k_part2 PREHOOK: type: QUERY PREHOOK: Input: default@over1k_part2 PREHOOK: Input: default@over1k_part2@ds=foo/t=27 PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### -POSTHOOK: query: select * from over1k_part2 +POSTHOOK: query: -- SORT_BEFORE_DIFF +select * from over1k_part2 POSTHOOK: type: QUERY POSTHOOK: Input: default@over1k_part2 POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 @@ -1800,13 +1802,15 @@ Bucket Columns: [] Sort Columns: [] Storage Desc Params: serialization.format 1 -PREHOOK: query: select * from over1k_part2 +PREHOOK: query: -- SORT_BEFORE_DIFF +select * from over1k_part2 PREHOOK: type: QUERY PREHOOK: Input: default@over1k_part2 PREHOOK: Input: default@over1k_part2@ds=foo/t=27 PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### -POSTHOOK: query: select * from over1k_part2 +POSTHOOK: query: -- SORT_BEFORE_DIFF +select * from over1k_part2 POSTHOOK: type: QUERY POSTHOOK: Input: default@over1k_part2 POSTHOOK: Input: default@over1k_part2@ds=foo/t=27