diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java index 24eeb9d..4dd17c1e 100644 --- itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java +++ itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java @@ -22,15 +22,15 @@ import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hive.beeline.Parallelized; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -@RunWith(Parameterized.class) +@RunWith(Parallelized.class) public class TestBeeLineDriver { static CliAdapter adapter = new CliConfigs.BeeLineConfig().getCliAdapter(); diff --git itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java new file mode 100644 index 0000000..a819e26 --- /dev/null +++ itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline; + +import org.junit.runners.Parameterized; +import org.junit.runners.model.RunnerScheduler; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * Class to run Parameterized test in parallel. + * Source: http://hwellmann.blogspot.hu/2009/12/running-parameterized-junit-tests-in.html + */ +public class Parallelized extends Parameterized { + private static class ThreadPoolScheduler implements RunnerScheduler { + private ExecutorService executor; + + public ThreadPoolScheduler() { + String threads = System.getProperty("junit.parallel.threads"); + int numThreads = Runtime.getRuntime().availableProcessors(); + if (threads != null) { + numThreads = Integer.parseInt(threads); + } + executor = Executors.newFixedThreadPool(numThreads); + } + + @Override + public void finished() { + executor.shutdown(); + try { + executor.awaitTermination(10, TimeUnit.MINUTES); + } catch (InterruptedException exc) { + throw new RuntimeException(exc); + } + } + + @Override + public void schedule(Runnable childStatement) { + executor.submit(childStatement); + } + } + + public Parallelized(Class klass) throws Throwable { + super(klass); + setScheduler(new ThreadPoolScheduler()); + } +} diff --git itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java index 49d6d24..d17b81c 100644 --- itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java +++ itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java @@ -188,6 +188,8 @@ private static RegexFilterSet getStaticFilterSet() { return new RegexFilterSet() .addFilter(logPattern, "") + .addFilter("(?s)\nWaiting to acquire compile lock:.*?Acquired the compile lock.\n", "\n") + .addFilter("Acquired the compile lock.\n", "") .addFilter("Getting log thread is interrupted, since query is done!\n", "") .addFilter("going to print operations logs\n", "") .addFilter("printed operations logs\n", "") diff --git ql/src/test/results/clientpositive/beeline/drop_with_concurrency.q.out ql/src/test/results/clientpositive/beeline/drop_with_concurrency.q.out index d22c9ec..993329e 100644 --- ql/src/test/results/clientpositive/beeline/drop_with_concurrency.q.out +++ ql/src/test/results/clientpositive/beeline/drop_with_concurrency.q.out @@ -9,7 +9,6 @@ No rows affected No rows affected >>> >>> drop table if exists drop_with_concurrency_1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): drop table if exists drop_with_concurrency_1 Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:null, properties:null) @@ -25,7 +24,6 @@ OK Shutting down query drop table if exists drop_with_concurrency_1 No rows affected >>> create table drop_with_concurrency_1 (c1 int); -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): create table drop_with_concurrency_1 (c1 int) Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:null, properties:null) @@ -45,7 +43,6 @@ OK Shutting down query create table drop_with_concurrency_1 (c1 int) No rows affected >>> drop table drop_with_concurrency_1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): drop table drop_with_concurrency_1 Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:null, properties:null) diff --git ql/src/test/results/clientpositive/beeline/escape_comments.q.out ql/src/test/results/clientpositive/beeline/escape_comments.q.out index 5f9df93..2a05e53 100644 --- ql/src/test/results/clientpositive/beeline/escape_comments.q.out +++ ql/src/test/results/clientpositive/beeline/escape_comments.q.out @@ -1,6 +1,5 @@ >>> !run !!{qFileDirectory}!!/escape_comments.q >>> create database escape_comments_db comment 'a\nb'; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): create database escape_comments_db comment 'a\nb' Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:null, properties:null) @@ -18,7 +17,6 @@ OK Shutting down query create database escape_comments_db comment 'a\nb' No rows affected >>> use escape_comments_db; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): use escape_comments_db Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:null, properties:null) @@ -38,7 +36,6 @@ No rows affected >>> create table escape_comments_tbl1 (col1 string comment 'a\nb\'\;') comment 'a\nb' partitioned by (p1 string comment 'a\nb'); -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): create table escape_comments_tbl1 (col1 string comment 'a\nb\'\;') comment 'a\nb' partitioned by (p1 string comment 'a\nb') @@ -69,7 +66,6 @@ partitioned by (p1 string comment 'a\nb') No rows affected >>> create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' as select col1 from escape_comments_tbl1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): create view escape_comments_view1 (col1 comment 'a\nb') comment 'a\nb' as select col1 from escape_comments_tbl1 Semantic Analysis Completed @@ -98,7 +94,6 @@ Shutting down query create view escape_comments_view1 (col1 comment 'a\nb') comm as select col1 from escape_comments_tbl1 No rows affected >>> create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb'; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): create index index2 on table escape_comments_tbl1(col1) as 'COMPACT' with deferred rebuild comment 'a\nb' Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:null, properties:null) @@ -118,7 +113,6 @@ Shutting down query create index index2 on table escape_comments_tbl1(col1) as ' No rows affected >>> >>> describe database extended escape_comments_db; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): describe database extended escape_comments_db Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:db_name, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer), FieldSchema(name:location, type:string, comment:from deserializer), FieldSchema(name:owner_name, type:string, comment:from deserializer), FieldSchema(name:owner_type, type:string, comment:from deserializer), FieldSchema(name:parameters, type:string, comment:from deserializer)], properties:null) @@ -138,7 +132,6 @@ Shutting down query describe database extended escape_comments_db 'escape_comments_db','a\nb','location/in/test','user','USER','' 1 row selected >>> describe database escape_comments_db; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): describe database escape_comments_db Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:db_name, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer), FieldSchema(name:location, type:string, comment:from deserializer), FieldSchema(name:owner_name, type:string, comment:from deserializer), FieldSchema(name:owner_type, type:string, comment:from deserializer), FieldSchema(name:parameters, type:string, comment:from deserializer)], properties:null) @@ -158,7 +151,6 @@ Shutting down query describe database escape_comments_db 'escape_comments_db','a\nb','location/in/test','user','USER','' 1 row selected >>> show create table escape_comments_tbl1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): show create table escape_comments_tbl1 Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:createtab_stmt, type:string, comment:from deserializer)], properties:null) @@ -192,7 +184,6 @@ Shutting down query show create table escape_comments_tbl1 ' 'transient_lastDdlTime'='!!UNIXTIME!!')' 15 rows selected >>> describe formatted escape_comments_tbl1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): describe formatted escape_comments_tbl1 Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col_name, type:string, comment:from deserializer), FieldSchema(name:data_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) @@ -247,7 +238,6 @@ Shutting down query describe formatted escape_comments_tbl1 '','serialization.format','1 ' 36 rows selected >>> describe pretty escape_comments_tbl1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): describe pretty escape_comments_tbl1 Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col_name, type:string, comment:from deserializer), FieldSchema(name:data_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) @@ -278,7 +268,6 @@ Shutting down query describe pretty escape_comments_tbl1 ' ',' ','b' 12 rows selected >>> describe escape_comments_tbl1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): describe escape_comments_tbl1 Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col_name, type:string, comment:from deserializer), FieldSchema(name:data_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) @@ -304,7 +293,6 @@ Shutting down query describe escape_comments_tbl1 'p1','string','a\nb' 7 rows selected >>> show create table escape_comments_view1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): show create table escape_comments_view1 Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:createtab_stmt, type:string, comment:from deserializer)], properties:null) @@ -324,7 +312,6 @@ Shutting down query show create table escape_comments_view1 'CREATE VIEW `escape_comments_view1` AS SELECT `col1` AS `col1` FROM (select `escape_comments_tbl1`.`col1` from `escape_comments_db`.`escape_comments_tbl1`) `escape_comments_db.escape_comments_view1`' 1 row selected >>> describe formatted escape_comments_view1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): describe formatted escape_comments_view1 Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:col_name, type:string, comment:from deserializer), FieldSchema(name:data_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) @@ -371,7 +358,6 @@ Shutting down query describe formatted escape_comments_view1 'View Rewrite Enabled:','No ','NULL' 28 rows selected >>> show formatted index on escape_comments_tbl1; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): show formatted index on escape_comments_tbl1 Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:[FieldSchema(name:idx_name, type:string, comment:from deserializer), FieldSchema(name:tab_name, type:string, comment:from deserializer), FieldSchema(name:col_names, type:string, comment:from deserializer), FieldSchema(name:idx_tab_name, type:string, comment:from deserializer), FieldSchema(name:idx_type, type:string, comment:from deserializer), FieldSchema(name:comment, type:string, comment:from deserializer)], properties:null) @@ -393,7 +379,6 @@ Shutting down query show formatted index on escape_comments_tbl1 4 rows selected >>> >>> drop database escape_comments_db cascade; -Acquired the compile lock. Compiling commandqueryId=(!!{queryId}!!): drop database escape_comments_db cascade Semantic Analysis Completed Returning Hive schema: Schema(fieldSchemas:null, properties:null)