Details

    • Type: Sub-task
    • Status: Closed
    • Priority: Critical
    • Resolution: Fixed
    • Affects Version/s: 0.11.0
    • Fix Version/s: 0.12.0
    • Component/s: None
    • Labels:
      None

      Description

      It appears our use of DataNucleaus is not correct or perhaps there is a bug in the ancient version of DN we are using. On startup having multiple threads performing "show tables" results in failures. Additionally concurrent DML will fail event after startup. I used the program below to demonstrate this.

      package org.apache.hadoop.hive.ql;
      
      import java.sql.Connection;
      import java.sql.DriverManager;
      import java.sql.Statement;
      import java.util.concurrent.ExecutorService;
      import java.util.concurrent.Executors;
      
      import org.apache.hive.jdbc.HiveDriver;
      
      public class MultiThreadTest {
      
        public static class QueryRunner implements Runnable {
          int id;
          double averageElapsedTime;
          Connection connection;
          Statement statement;
          QueryRunner(int id) {
            this.id = id;
          }
      
          @Override
          public void run() {
            long count = 0;
            double elapsedTime = 0;
            try {
              connection = DriverManager.getConnection("jdbc:hive2://localhost:10000/default", "brock", "password");
              statement = connection.createStatement();
      //        statement.execute("DROP TABLE IF EXISTS t" + id);
              for (int i = 0; i < 10; i++) {
      //          statement.execute("CREATE TABLE t" + id + " (key int)");
                long start = System.currentTimeMillis();
      //          statement.execute("DROP TABLE t" + id);
                statement.execute("SHOW TABLES");
                elapsedTime += System.currentTimeMillis() - start;
                count++;
              }
              if(statement != null) {
                statement.close();
              }
              if(connection != null) {
                connection.close();
              }
            } catch (Exception e) {
              e.printStackTrace();
            } finally {
              if(count > 0) {
                averageElapsedTime = elapsedTime / (double)count;
              }
            }
          }
        }
      
        public static void main(String[] args) throws Exception {
          int numThreads = 50;
          Class.forName(HiveDriver.class.getName());
          ExecutorService executor = Executors.newFixedThreadPool(numThreads);
          QueryRunner[] queryRunners = new QueryRunner[numThreads];
          for (int i = 0; i < numThreads; i++) {
            queryRunners[i] = new QueryRunner(i);
            executor.execute(queryRunners[i]);
          }
          executor.shutdown();
          while(!executor.isTerminated()) {
            System.out.println("Waiting...");
            Thread.sleep(1000L);
          }
          for (int i = 0; i < numThreads; i++) {
            System.out.println(Math.round(queryRunners[i].averageElapsedTime));
          }
        }
      }
      

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                brocknoland Brock Noland
                Reporter:
                brocknoland Brock Noland
              • Votes:
                0 Vote for this issue
                Watchers:
                2 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: