diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2f695d4..c0f0663 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4834,6 +4834,8 @@ HIVE_SECURITY_AUTHORIZATION_SCHEDULED_QUERIES_SUPPORTED("hive.security.authorization.scheduled.queries.supported", false, "Enable this if the configured authorizer is able to handle scheduled query related calls."), + HIVE_SCHEDULED_QUERIES_MAX_EXECUTORS("hive.scheduled.queries.max.executors", "4", + "Maximal number of scheduled query executors."), HIVE_QUERY_RESULTS_CACHE_ENABLED("hive.query.results.cache.enabled", true, "If the query results cache is enabled. This will keep results of previously executed queries " + diff --git metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql index fde6f02..03540bb 100644 --- metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql +++ metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql @@ -1211,6 +1211,7 @@ `USER` string, `QUERY` string, `NEXT_EXECUTION` bigint, + `ACTIVE_EXECUTION_ID` bigint, CONSTRAINT `SYS_PK_SCHEDULED_QUERIES` PRIMARY KEY (`SCHEDULED_QUERY_ID`) DISABLE ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' @@ -1225,7 +1226,8 @@ \"SCHEDULE\", \"USER\", \"QUERY\", - \"NEXT_EXECUTION\" + \"NEXT_EXECUTION\", + \"ACTIVE_EXECUTION_ID\" FROM \"SCHEDULED_QUERIES\"" ); @@ -1795,7 +1797,8 @@ `SCHEDULE`, `USER`, `QUERY`, - FROM_UNIXTIME(NEXT_EXECUTION) as NEXT_EXECUTION + FROM_UNIXTIME(NEXT_EXECUTION) as NEXT_EXECUTION, + `ACTIVE_EXECUTION_ID` FROM SYS.SCHEDULED_QUERIES ; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/schq/ScheduledQueryMaintenanceTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/schq/ScheduledQueryMaintenanceTask.java index fd0c173..fe5be47 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/schq/ScheduledQueryMaintenanceTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/schq/ScheduledQueryMaintenanceTask.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.exec.schq; import org.apache.hadoop.hive.metastore.api.ScheduledQueryMaintenanceRequest; +import org.apache.hadoop.hive.metastore.api.ScheduledQueryMaintenanceRequestType; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -46,7 +47,8 @@ ScheduledQueryMaintenanceRequest request = buildScheduledQueryRequest(); try { Hive.get().getMSC().scheduledQueryMaintenance(request); - if (work.getScheduledQuery().isSetNextExecution()) { + if (work.getScheduledQuery().isSetNextExecution() + || request.getType() == ScheduledQueryMaintenanceRequestType.CREATE) { ScheduledQueryExecutionService.forceScheduleCheck(); } } catch (TException | HiveException e) { diff --git ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionContext.java ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionContext.java index 1bb24ee..32cb316 100644 --- ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionContext.java @@ -66,4 +66,8 @@ return conf.getTimeVar(ConfVars.HIVE_SCHEDULED_QUERIES_EXECUTOR_PROGRESS_REPORT_INTERVAL, TimeUnit.MILLISECONDS); } + public int getNumberOfExecutors() { + return conf.getIntVar(ConfVars.HIVE_SCHEDULED_QUERIES_MAX_EXECUTORS); + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionService.java ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionService.java index 9a6237c..7cc3438 100644 --- ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionService.java +++ ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionService.java @@ -19,6 +19,8 @@ import java.io.Closeable; import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -48,8 +50,10 @@ private static ScheduledQueryExecutionService INSTANCE = null; private ScheduledQueryExecutionContext context; - private ScheduledQueryExecutor worker; private AtomicInteger forcedScheduleCheckCounter = new AtomicInteger(); + private ScheduledQueryPoller poller; + AtomicInteger usedExecutors = new AtomicInteger(0); + List runningWorkers = new LinkedList<>(); public static ScheduledQueryExecutionService startScheduledQueryExecutorService(HiveConf inputConf) { HiveConf conf = new HiveConf(inputConf); @@ -73,7 +77,7 @@ private ScheduledQueryExecutionService(ScheduledQueryExecutionContext ctx) { context = ctx; - ctx.executor.submit(worker = new ScheduledQueryExecutor()); + ctx.executor.submit(poller = new ScheduledQueryPoller()); ctx.executor.submit(new ProgressReporter()); } @@ -81,27 +85,28 @@ return state == QueryState.FINISHED || state == QueryState.FAILED; } - class ScheduledQueryExecutor implements Runnable { - - private ScheduledQueryProgressInfo info; + class ScheduledQueryPoller implements Runnable { @Override public void run() { while (true) { - ScheduledQueryPollResponse q = context.schedulerService.scheduledQueryPoll(); - if (q.isSetExecutionId()) { - try{ - processQuery(q); - } catch (Throwable t) { - LOG.error("Unexpected exception during scheduled query processing", t); - } - } else { + if (usedExecutors.get() < context.getNumberOfExecutors()) { try { - sleep(context.getIdleSleepTime()); - } catch (InterruptedException e) { - LOG.warn("interrupt discarded"); + ScheduledQueryPollResponse q = context.schedulerService.scheduledQueryPoll(); + if (q.isSetExecutionId()) { + context.executor.submit(new ScheduledQueryExecutor(q)); + // skip sleep and poll again if there are available executor + continue; + } + } catch (Throwable t) { + LOG.error("Unexpected exception during scheduled query submission", t); } } + try { + sleep(context.getIdleSleepTime()); + } catch (InterruptedException e) { + LOG.warn("interrupt discarded"); + } } } @@ -116,6 +121,37 @@ } } + } + + private void workerStarted(ScheduledQueryExecutor executor) { + runningWorkers.add(executor); + usedExecutors.incrementAndGet(); + } + + private void workerStopped(ScheduledQueryExecutor executor) { + usedExecutors.decrementAndGet(); + runningWorkers.remove(executor); + forceScheduleCheck(); + } + + class ScheduledQueryExecutor implements Runnable { + + private ScheduledQueryProgressInfo info; + final ScheduledQueryPollResponse pollResponse; + + public ScheduledQueryExecutor(ScheduledQueryPollResponse pollResponse) { + workerStarted(this); + this.pollResponse = pollResponse; + } + + public void run() { + try { + processQuery(pollResponse); + } finally { + workerStopped(this); + } + } + public synchronized void reportQueryProgress() { if (info != null) { LOG.info("Reporting query progress of {} as {} err:{}", info.getScheduledExecutionId(), info.getState(), @@ -190,7 +226,9 @@ LOG.warn("interrupt discarded"); } try { - worker.reportQueryProgress(); + for (ScheduledQueryExecutor worker : runningWorkers) { + worker.reportQueryProgress(); + } } catch (Exception e) { LOG.error("ProgressReporter encountered exception ", e); } diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetastoreTaskThread.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetastoreTaskThread.java index e5d21b0..d56bc2a 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetastoreTaskThread.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetastoreTaskThread.java @@ -19,7 +19,6 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; - import java.util.concurrent.TimeUnit; /** @@ -35,4 +34,13 @@ * @return frequency */ long runFrequency(TimeUnit unit); + + /** + * Gets the initial delay before the first execution. + * + * Defaults to {@link #runFrequency(TimeUnit)} + */ + default long initialDelay(TimeUnit unit) { + return runFrequency(unit); + } } diff --git standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/utils/package-info.java standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/utils/package-info.java deleted file mode 100644 index 2eb51c8..0000000 --- standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/utils/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Package consisting the utility methods for metastore. - */ -package org.apache.hadoop.hive.metastore.utils; diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 1a5944d..8a826d2 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -12667,7 +12667,7 @@ try { openTransaction(); Query q = pm.newQuery(MScheduledQuery.class, - "nextExecution <= now && enabled && clusterNamespace == ns"); + "nextExecution <= now && enabled && clusterNamespace == ns && activeExecution == null"); q.setSerializeRead(true); q.declareParameters("java.lang.Integer now, java.lang.String ns"); q.setOrdering("nextExecution"); @@ -12685,6 +12685,7 @@ execution.setState(QueryState.INITED); execution.setStartTime(now); execution.setLastUpdateTime(now); + schq.setActiveExecution(execution); pm.makePersistent(execution); pm.makePersistent(schq); ObjectStoreTestHook.onScheduledQueryPoll(); @@ -12735,6 +12736,7 @@ case TIMED_OUT: execution.setEndTime((int) (System.currentTimeMillis() / 1000)); execution.setLastUpdateTime(null); + execution.getScheduledQuery().setActiveExecution(null); break; default: throw new InvalidOperationException("invalid state: " + info.getState()); @@ -12967,6 +12969,8 @@ // info.set scheduledQueryProgress(info); } + + recoverInvalidScheduledQueryState(timeoutSecs); committed = commitTransaction(); return results.size(); } finally { @@ -12975,4 +12979,21 @@ } } } + + private void recoverInvalidScheduledQueryState(int timeoutSecs) { + int maxLastUpdateTime = (int) (System.currentTimeMillis() / 1000) - timeoutSecs; + Query q = pm.newQuery(MScheduledQuery.class); + q.setFilter("activeExecution != null"); + + List results = (List) q.execute(); + for (MScheduledQuery e : results) { + Integer lastUpdateTime = e.getActiveExecution().getLastUpdateTime(); + if (lastUpdateTime == null || lastUpdateTime < maxLastUpdateTime) { + LOG.error("Scheduled query: {} stuck with an activeExecution - clearing", + scheduledQueryKeyRef(e.getScheduleKey())); + e.setActiveExecution(null); + pm.makePersistent(e); + } + } + } } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ScheduledQueryExecutionsMaintTask.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ScheduledQueryExecutionsMaintTask.java index d678d01..4c1b34d 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ScheduledQueryExecutionsMaintTask.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ScheduledQueryExecutionsMaintTask.java @@ -35,6 +35,13 @@ private Configuration conf; @Override + public long initialDelay(TimeUnit unit) { + // no delay before the first execution; + // after an ungracefull shutdown it might take time to notice that in-flight scheduled queries are not running anymore + return 0; + } + + @Override public long runFrequency(TimeUnit unit) { return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.SCHEDULED_QUERIES_EXECUTION_MAINT_TASK_FREQUENCY, unit); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MScheduledQuery.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MScheduledQuery.java index d055e7d..c80241b 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MScheduledQuery.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MScheduledQuery.java @@ -35,6 +35,7 @@ private String user; private String query; private Integer nextExecution; + private MScheduledExecution activeExecution; private Set executions; public MScheduledQuery(ScheduledQuery s) { @@ -112,4 +113,12 @@ return user; } + public void setActiveExecution(MScheduledExecution execution) { + activeExecution = execution; + } + + public MScheduledExecution getActiveExecution() { + return activeExecution; + } + } diff --git standalone-metastore/metastore-server/src/main/resources/package.jdo standalone-metastore/metastore-server/src/main/resources/package.jdo index ce919e4..88eabfa 100644 --- standalone-metastore/metastore-server/src/main/resources/package.jdo +++ standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -1471,6 +1471,10 @@ + + + + diff --git standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql index 482d8aa..e9687ab 100644 --- standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -747,7 +747,8 @@ "USER" varchar(128) not null, "SCHEDULE" varchar(256) not null, "QUERY" varchar(4000) not null, - "NEXT_EXECUTION" integer not null + "NEXT_EXECUTION" integer not null, + "ACTIVE_EXECUTION_ID" bigint ); CREATE INDEX NEXTEXECUTIONINDEX ON APP.SCHEDULED_QUERIES (ENABLED,CLUSTER_NAMESPACE,NEXT_EXECUTION); diff --git standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql index 6fd8ae6..507756c 100644 --- standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql +++ standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql @@ -56,6 +56,9 @@ ALTER TABLE "APP"."KEY_CONSTRAINTS" DROP CONSTRAINT "CONSTRAINTS_PK"; ALTER TABLE "APP"."KEY_CONSTRAINTS" ADD CONSTRAINT "CONSTRAINTS_PK" PRIMARY KEY ("PARENT_TBL_ID", "CONSTRAINT_NAME", "POSITION"); +-- HIVE-22872 +ALTER TABLE "SCHEDULED_QUERIES" ADD "ACTIVE_EXECUTION_ID" bigint; + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index dfd4bd6..e347bf1 100644 --- standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -1290,6 +1290,35 @@ INSERT INTO SEQUENCE_TABLE (SEQUENCE_NAME, NEXT_VAL) VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1); +CREATE TABLE "SCHEDULED_QUERIES" ( + "SCHEDULED_QUERY_ID" bigint NOT NULL, + "CLUSTER_NAMESPACE" VARCHAR(256), + "ENABLED" bit NOT NULL DEFAULT 0, + "NEXT_EXECUTION" INTEGER, + "QUERY" VARCHAR(4000), + "SCHEDULE" VARCHAR(256), + "SCHEDULE_NAME" VARCHAR(256), + "USER" VARCHAR(256), + "ACTIVE_EXECUTION_ID" bigint, + CONSTRAINT SCHEDULED_QUERIES_PK PRIMARY KEY ("SCHEDULED_QUERY_ID") +); + +CREATE TABLE "SCHEDULED_EXECUTIONS" ( + "SCHEDULED_EXECUTION_ID" bigint NOT NULL, + "END_TIME" INTEGER, + "ERROR_MESSAGE" VARCHAR(2000), + "EXECUTOR_QUERY_ID" VARCHAR(256), + "LAST_UPDATE_TIME" INTEGER, + "SCHEDULED_QUERY_ID" bigint, + "START_TIME" INTEGER, + "STATE" VARCHAR(256), + CONSTRAINT SCHEDULED_EXECUTIONS_PK PRIMARY KEY ("SCHEDULED_EXECUTION_ID"), + CONSTRAINT SCHEDULED_EXECUTIONS_SCHQ_FK FOREIGN KEY ("SCHEDULED_QUERY_ID") REFERENCES "SCHEDULED_QUERIES"("SCHEDULED_QUERY_ID") ON DELETE CASCADE +); + +CREATE INDEX IDX_SCHEDULED_EX_LAST_UPDATE ON "SCHEDULED_EXECUTIONS" ("LAST_UPDATE_TIME"); +CREATE INDEX IDX_SCHEDULED_EX_SQ_ID ON "SCHEDULED_EXECUTIONS" ("SCHEDULED_QUERY_ID"); + -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script -- ----------------------------------------------------------------- diff --git standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql index fecfca8..4cf0a6e 100644 --- standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql +++ standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql @@ -30,6 +30,36 @@ ALTER TABLE KEY_CONSTRAINTS DROP CONSTRAINT CONSTRAINTS_PK; ALTER TABLE KEY_CONSTRAINTS ADD CONSTRAINT CONSTRAINTS_PK PRIMARY KEY (PARENT_TBL_ID, CONSTRAINT_NAME, POSITION); +CREATE TABLE "SCHEDULED_QUERIES" ( + "SCHEDULED_QUERY_ID" bigint NOT NULL, + "CLUSTER_NAMESPACE" VARCHAR(256), + "ENABLED" bit NOT NULL DEFAULT 0, + "NEXT_EXECUTION" INTEGER, + "QUERY" VARCHAR(4000), + "SCHEDULE" VARCHAR(256), + "SCHEDULE_NAME" VARCHAR(256), + "USER" VARCHAR(256), + "ACTIVE_EXECUTION_ID" bigint, + CONSTRAINT SCHEDULED_QUERIES_PK PRIMARY KEY ("SCHEDULED_QUERY_ID") +); + +CREATE TABLE "SCHEDULED_EXECUTIONS" ( + "SCHEDULED_EXECUTION_ID" bigint NOT NULL, + "END_TIME" INTEGER, + "ERROR_MESSAGE" VARCHAR(2000), + "EXECUTOR_QUERY_ID" VARCHAR(256), + "LAST_UPDATE_TIME" INTEGER, + "SCHEDULED_QUERY_ID" bigint, + "START_TIME" INTEGER, + "STATE" VARCHAR(256), + CONSTRAINT SCHEDULED_EXECUTIONS_PK PRIMARY KEY ("SCHEDULED_EXECUTION_ID"), + CONSTRAINT SCHEDULED_EXECUTIONS_SCHQ_FK FOREIGN KEY ("SCHEDULED_QUERY_ID") REFERENCES "SCHEDULED_QUERIES"("SCHEDULED_QUERY_ID") ON DELETE CASCADE +); + +CREATE INDEX IDX_SCHEDULED_EX_LAST_UPDATE ON "SCHEDULED_EXECUTIONS" ("LAST_UPDATE_TIME"); +CREATE INDEX IDX_SCHEDULED_EX_SQ_ID ON "SCHEDULED_EXECUTIONS" ("SCHEDULED_QUERY_ID"); + + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS MESSAGE; diff --git standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index a9a0930..e656386 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -1222,6 +1222,7 @@ SCHEDULE VARCHAR(256), SCHEDULE_NAME VARCHAR(256), `USER` VARCHAR(256), + ACTIVE_EXECUTION_ID INTEGER, CONSTRAINT SCHEDULED_QUERIES_PK PRIMARY KEY (SCHEDULED_QUERY_ID) ); diff --git standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql index 70b5d9d..0110d12 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql @@ -60,6 +60,9 @@ ALTER TABLE `KEY_CONSTRAINTS` DROP PRIMARY KEY; ALTER TABLE `KEY_CONSTRAINTS` ADD CONSTRAINT `CONSTRAINTS_PK` PRIMARY KEY (`PARENT_TBL_ID`, `CONSTRAINT_NAME`, `POSITION`); +-- HIVE-22872 +ALTER TABLE SCHEDULED_QUERIES ADD COLUMN ACTIVE_EXECUTION_ID INTEGER ; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS MESSAGE; diff --git standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index f90d76b..ce2c380 100644 --- standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -1197,6 +1197,7 @@ "SCHEDULE" VARCHAR(256), "SCHEDULE_NAME" VARCHAR(256), "USER" VARCHAR(256), + "ACTIVE_EXECUTION_ID" number(19), CONSTRAINT SCHEDULED_QUERIES_PK PRIMARY KEY ("SCHEDULED_QUERY_ID") ); diff --git standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql index 9f1b980..10cca89 100644 --- standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql +++ standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql @@ -60,6 +60,9 @@ ALTER TABLE KEY_CONSTRAINTS DROP CONSTRAINT CONSTRAINTS_PK; ALTER TABLE KEY_CONSTRAINTS ADD CONSTRAINT CONSTRAINTS_PK PRIMARY KEY (PARENT_TBL_ID, CONSTRAINT_NAME, POSITION); +-- HIVE-22872 +ALTER TABLE SCHEDULED_QUERIES ADD ACTIVE_EXECUTION_ID number(19); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS Status from dual; diff --git standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index b992d7a..f5ffc68 100644 --- standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -1885,6 +1885,7 @@ "SCHEDULE" VARCHAR(256), "SCHEDULE_NAME" VARCHAR(256), "USER" VARCHAR(256), + "ACTIVE_EXECUTION_ID" BIGINT, CONSTRAINT "SCHEDULED_QUERIES_PK" PRIMARY KEY ("SCHEDULED_QUERY_ID") ); diff --git standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql index f9af248..6e1df95 100644 --- standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql +++ standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql @@ -191,6 +191,9 @@ ALTER TABLE "KEY_CONSTRAINTS" DROP CONSTRAINT "KEY_CONSTRAINTS_pkey"; ALTER TABLE "KEY_CONSTRAINTS" ADD CONSTRAINT "CONSTRAINTS_PK" PRIMARY KEY ("PARENT_TBL_ID", "CONSTRAINT_NAME", "POSITION"); +-- HIVE-22872 +ALTER TABLE "SCHEDULED_QUERIES" ADD "ACTIVE_EXECUTION_ID" bigint; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0';