diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index ed6d3d80e3f6a9ba66f46223d97666b48c388028..5954bb9757e318ab75eed6e353be5b578a9b3bfe 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3205,6 +3205,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_SERVER2_TEZ_INTERACTIVE_QUEUE("hive.server2.tez.interactive.queue", "", "A single YARN queues to use for Hive Interactive sessions. When this is specified,\n" + "workload management is enabled and used for these sessions."), + HIVE_SERVER2_WM_NAMESPACE("hive.server2.wm.namespace", "default", + "The WM namespace to use when one metastore is used by multiple compute clusters each \n" + + "with their own workload management. The special value 'default' (the default) will \n" + + "also include any resource plans created before the namespaces were introduced."), HIVE_SERVER2_WM_WORKER_THREADS("hive.server2.wm.worker.threads", 4, "Number of worker threads to use to perform the synchronous operations with Tez\n" + "sessions for workload management (e.g. opening, closing, etc.)"), diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index c3e1e8e88c69d7713e16c7061ce8cf73a0d5e833..d9fb645858887feba44e49e7cc91d98327b4f3c1 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -1105,36 +1105,36 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int } @Override - public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException, MetaException { - return objectStore.getResourcePlan(name); + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + return objectStore.getResourcePlan(name, ns); } @Override - public List getAllResourcePlans() throws MetaException { - return objectStore.getAllResourcePlans(); + public List getAllResourcePlans(String ns) throws MetaException { + return objectStore.getAllResourcePlans(ns); } @Override - public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - return objectStore.alterResourcePlan(name, resourcePlan, canActivateDisabled, canDeactivate, isReplace); + return objectStore.alterResourcePlan(name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { - return objectStore.getActiveResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { + return objectStore.getActiveResourcePlan(ns); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { - return objectStore.validateResourcePlan(name); + return objectStore.validateResourcePlan(name, ns); } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { - objectStore.dropResourcePlan(name); + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + objectStore.dropResourcePlan(name, ns); } @Override @@ -1151,15 +1151,15 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMTrigger(resourcePlanName, triggerName); + objectStore.dropWMTrigger(resourcePlanName, triggerName, ns); } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { - return objectStore.getTriggersForResourcePlan(resourcePlanName); + return objectStore.getTriggersForResourcePlan(resourcePlanName, ns); } @Override @@ -1175,9 +1175,9 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMPool(resourcePlanName, poolPath); + objectStore.dropWMPool(resourcePlanName, poolPath, ns); } @Override @@ -1195,15 +1195,15 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + objectStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { + objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } @Override diff --git metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql index a69046f961cdf0fff7989492c489bb62f2a66d72..9d6bec02432a97551a071ef1f6cd2ae744d6d573 100644 --- metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql +++ metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql @@ -941,6 +941,7 @@ FROM `PARTITION_PARAMS` GROUP BY `PART_ID`; CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( `NAME` string, + `NS` string, `STATUS` string, `QUERY_PARALLELISM` int, `DEFAULT_POOL_PATH` string @@ -951,6 +952,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end, \"STATUS\", \"WM_RESOURCEPLAN\".\"QUERY_PARALLELISM\", \"WM_POOL\".\"PATH\" diff --git metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql index 4c770206fe3dcceb8570be1c1ef078b376f5cafd..70f093a4ad4e2e306c1498cf29aaf73c6caaf2e5 100644 --- metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql +++ metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql @@ -2,6 +2,31 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0'; USE SYS; +-- HIVE-20793 +DROP TABLE IF EXISTS `WM_RESOURCEPLANS`; +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( + `NAME` string, + `NS` string, + `STATUS` string, + `QUERY_PARALLELISM` int, + `DEFAULT_POOL_PATH` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end, + \"STATUS\", + \"WM_RESOURCEPLAN\".\"QUERY_PARALLELISM\", + \"WM_POOL\".\"PATH\" +FROM + \"WM_RESOURCEPLAN\" LEFT OUTER JOIN \"WM_POOL\" ON \"WM_RESOURCEPLAN\".\"DEFAULT_POOL_ID\" = \"WM_POOL\".\"POOL_ID\"" +); + + + DROP TABLE IF EXISTS `VERSION`; CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '4.0.0' AS `SCHEMA_VERSION`, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 807f159daa98d40e667914adc6c53fb8ecabf998..8eb9ebe607bad20f3b6c83e466242dc2959df83c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -695,6 +695,11 @@ private int showResourcePlans(Hive db, ShowResourcePlanDesc showResourcePlanDesc return 0; } + // Note: the resource plan operations are going to be annotated with namespace based on the config + // inside Hive.java. We don't want HS2 to be aware of namespaces beyond that, or to even see + // that there exist other namespaces, because one HS2 always operates inside just one and we + // don't want this complexity to bleed everywhere. Therefore, this code doesn't care about + // namespaces - Hive.java will transparently scope everything. That's the idea anyway. private int alterResourcePlan(Hive db, AlterResourcePlanDesc desc) throws HiveException { if (desc.shouldValidate()) { WMValidateResourcePlanResponse result = db.validateResourcePlan(desc.getResourcePlanName()); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 4de038913a5c9a2c199f71702b8f70ca84d0856b..a161426ae34088febbbcbeb7faeaf24d309bd0e7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -5444,9 +5444,15 @@ public void addCheckConstraint(List checkConstraints) } } - public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName, boolean ifNotExists) throws HiveException { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (resourcePlan.isSetNs() && !ns.equals(resourcePlan.getNs())) { + throw new HiveException("Cannot modify a plan in a different NS; was " + + resourcePlan.getNs() + ", configured " + ns); + } + resourcePlan.setNs(ns); + try { getMSC().createResourcePlan(resourcePlan, copyFromName); } catch (AlreadyExistsException e) { @@ -5460,7 +5466,7 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName, public WMFullResourcePlan getResourcePlan(String rpName) throws HiveException { try { - return getMSC().getResourcePlan(rpName); + return getMSC().getResourcePlan(rpName, conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (NoSuchObjectException e) { return null; } catch (Exception e) { @@ -5470,7 +5476,7 @@ public WMFullResourcePlan getResourcePlan(String rpName) throws HiveException { public List getAllResourcePlans() throws HiveException { try { - return getMSC().getAllResourcePlans(); + return getMSC().getAllResourcePlans(conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (Exception e) { throw new HiveException(e); } @@ -5478,7 +5484,8 @@ public WMFullResourcePlan getResourcePlan(String rpName) throws HiveException { public void dropResourcePlan(String rpName, boolean ifExists) throws HiveException { try { - getMSC().dropResourcePlan(rpName); + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + getMSC().dropResourcePlan(rpName, ns); } catch (NoSuchObjectException e) { if (!ifExists) { throw new HiveException(e, ErrorMsg.RESOURCE_PLAN_NOT_EXISTS, rpName); @@ -5491,7 +5498,13 @@ public void dropResourcePlan(String rpName, boolean ifExists) throws HiveExcepti public WMFullResourcePlan alterResourcePlan(String rpName, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) throws HiveException { try { - return getMSC().alterResourcePlan(rpName, resourcePlan, canActivateDisabled, + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (resourcePlan.isSetNs() && !ns.equals(resourcePlan.getNs())) { + throw new HiveException("Cannot modify a plan in a different NS; was " + + resourcePlan.getNs() + ", configured " + ns); + } + resourcePlan.setNs(ns); + return getMSC().alterResourcePlan(rpName, ns, resourcePlan, canActivateDisabled, isForceDeactivate, isReplace); } catch (Exception e) { throw new HiveException(e); @@ -5500,7 +5513,8 @@ public WMFullResourcePlan alterResourcePlan(String rpName, WMNullableResourcePla public WMFullResourcePlan getActiveResourcePlan() throws HiveException { try { - return getMSC().getActiveResourcePlan(); + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + return getMSC().getActiveResourcePlan(ns); } catch (Exception e) { throw new HiveException(e); } @@ -5508,7 +5522,8 @@ public WMFullResourcePlan getActiveResourcePlan() throws HiveException { public WMValidateResourcePlanResponse validateResourcePlan(String rpName) throws HiveException { try { - return getMSC().validateResourcePlan(rpName); + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + return getMSC().validateResourcePlan(rpName, ns); } catch (Exception e) { throw new HiveException(e); } @@ -5516,6 +5531,12 @@ public WMValidateResourcePlanResponse validateResourcePlan(String rpName) throws public void createWMTrigger(WMTrigger trigger) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (trigger.isSetNs() && !ns.equals(trigger.getNs())) { + throw new HiveException("Cannot modify a plan in a different NS; was " + + trigger.getNs() + ", configured " + ns); + } + trigger.setNs(ns); getMSC().createWMTrigger(trigger); } catch (Exception e) { throw new HiveException(e); @@ -5524,6 +5545,12 @@ public void createWMTrigger(WMTrigger trigger) throws HiveException { public void alterWMTrigger(WMTrigger trigger) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (trigger.isSetNs() && !ns.equals(trigger.getNs())) { + throw new HiveException("Cannot modify a plan in a different NS; was " + + trigger.getNs() + ", configured " + ns); + } + trigger.setNs(ns); getMSC().alterWMTrigger(trigger); } catch (Exception e) { throw new HiveException(e); @@ -5532,7 +5559,7 @@ public void alterWMTrigger(WMTrigger trigger) throws HiveException { public void dropWMTrigger(String rpName, String triggerName) throws HiveException { try { - getMSC().dropWMTrigger(rpName, triggerName); + getMSC().dropWMTrigger(rpName, triggerName, conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (Exception e) { throw new HiveException(e); } @@ -5540,6 +5567,12 @@ public void dropWMTrigger(String rpName, String triggerName) throws HiveExceptio public void createWMPool(WMPool pool) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (pool.isSetNs() && !ns.equals(pool.getNs())) { + throw new HiveException("Cannot modify a plan in a different NS; was " + + pool.getNs() + ", configured " + ns); + } + pool.setNs(ns); getMSC().createWMPool(pool); } catch (Exception e) { throw new HiveException(e); @@ -5548,6 +5581,12 @@ public void createWMPool(WMPool pool) throws HiveException { public void alterWMPool(WMNullablePool pool, String poolPath) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (pool.isSetNs() && !ns.equals(pool.getNs())) { + throw new HiveException("Cannot modify a plan in a different NS; was " + + pool.getNs() + ", configured " + ns); + } + pool.setNs(ns); getMSC().alterWMPool(pool, poolPath); } catch (Exception e) { throw new HiveException(e); @@ -5556,7 +5595,8 @@ public void alterWMPool(WMNullablePool pool, String poolPath) throws HiveExcepti public void dropWMPool(String resourcePlanName, String poolPath) throws HiveException { try { - getMSC().dropWMPool(resourcePlanName, poolPath); + getMSC().dropWMPool(resourcePlanName, poolPath, + conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (Exception e) { throw new HiveException(e); } @@ -5565,6 +5605,12 @@ public void dropWMPool(String resourcePlanName, String poolPath) throws HiveExce public void createOrUpdateWMMapping(WMMapping mapping, boolean isUpdate) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (mapping.isSetNs() && !ns.equals(mapping.getNs())) { + throw new HiveException("Cannot modify a plan in a different NS; was " + + mapping.getNs() + ", configured " + ns); + } + mapping.setNs(ns); getMSC().createOrUpdateWMMapping(mapping, isUpdate); } catch (Exception e) { throw new HiveException(e); @@ -5573,17 +5619,24 @@ public void createOrUpdateWMMapping(WMMapping mapping, boolean isUpdate) public void dropWMMapping(WMMapping mapping) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (mapping.isSetNs() && !ns.equals(mapping.getNs())) { + throw new HiveException("Cannot modify a plan in a different NS; was " + + mapping.getNs() + ", configured " + ns); + } + mapping.setNs(ns); getMSC().dropWMMapping(mapping); } catch (Exception e) { throw new HiveException(e); } } - + // TODO: eh public void createOrDropTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, boolean shouldDrop) throws HiveException { try { - getMSC().createOrDropTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, shouldDrop); + getMSC().createOrDropTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, + shouldDrop, conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (Exception e) { throw new HiveException(e); } diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index e57db935d9420508ed6091e12ca6b6cd3382db5d..f8c441076f1c88d8142b55ce5b913bcebe8c69a0 100755 --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -26,6 +26,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; import org.apache.hadoop.fs.FileStatus; @@ -37,7 +38,13 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMPool; +import org.apache.hadoop.hive.metastore.api.WMResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMResourcePlanStatus; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.session.SessionState; @@ -59,8 +66,10 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import org.apache.thrift.protocol.TBinaryProtocol; import org.junit.Assert; +import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import junit.framework.TestCase; @@ -76,20 +85,21 @@ protected void setUp() throws Exception { super.setUp(); hiveConf = new HiveConf(this.getClass()); - hiveConf - .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + hm = setUpImpl(hiveConf); + } + + private static Hive setUpImpl(HiveConf hiveConf) throws Exception { + hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); // enable trash so it can be tested hiveConf.setFloat("fs.trash.checkpoint.interval", 30); // FS_TRASH_CHECKPOINT_INTERVAL_KEY (hadoop-2) hiveConf.setFloat("fs.trash.interval", 30); // FS_TRASH_INTERVAL_KEY (hadoop-2) SessionState.start(hiveConf); try { - hm = Hive.get(hiveConf); + return Hive.get(hiveConf); } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); - System.err - .println("Unable to initialize Hive Metastore using configuration: \n " - + hiveConf); + System.err.println("Unable to initialize Hive Metastore using configuration: \n" + hiveConf); throw e; } } @@ -421,6 +431,53 @@ public void testGetAndDropTables() throws Throwable { throw e; } } + static final private org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger("WTF"); + public void testWmNamespaceHandling() throws Throwable { + HiveConf hiveConf = new HiveConf(this.getClass()); + Hive hm = setUpImpl(hiveConf); + // TODO: threadlocals... Why is all this Hive client stuff like that?!! + final AtomicReference hm2r = new AtomicReference<>(); + Thread pointlessThread = new Thread(new Runnable() { + @Override + public void run() { + HiveConf hiveConf2 = new HiveConf(this.getClass()); + hiveConf2.setVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE, "hm2"); + try { + hm2r.set(setUpImpl(hiveConf2)); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + } + } + }); + pointlessThread.start(); + pointlessThread.join(); + Hive hm2 = hm2r.get(); + assertNotNull(hm2); + + + hm.createResourcePlan(new WMResourcePlan("hm"), null, false); + assertEquals(1, hm.getAllResourcePlans().size()); + assertEquals(0, hm2.getAllResourcePlans().size()); + hm2.createResourcePlan(new WMResourcePlan("hm"), null, false); + WMNullableResourcePlan changes = new WMNullableResourcePlan(); + changes.setStatus(WMResourcePlanStatus.ACTIVE); + hm.alterResourcePlan("hm", changes, true, false, false); + // We should not be able to modify the active plan. + WMPool pool = new WMPool("hm", "foo"); + pool.setAllocFraction(0); + pool.setQueryParallelism(1); + try { + hm.createWMPool(pool); + fail("Expected exception"); + } catch (HiveException e) { + } + // But we should still be able to modify the other plan. + pool.unsetNs(); // The call to create sets the namespace. + hm2.createWMPool(pool); + // Make the 2nd plan active in a different namespace. + changes.unsetNs(); + hm2.alterResourcePlan("hm", changes, true, false, false); + } public void testDropTableTrash() throws Throwable { if (!ShimLoader.getHadoopShims().supportTrashFeature()) { diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index aba63f050b5b98a2aeeb0df6ff2de5e6e06761f2..3ebfbfa1b5147c65201035b7e186cd5b1be7e4d5 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -51,6 +51,7 @@ import javax.security.auth.login.LoginException; import com.google.common.base.Preconditions; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -3413,34 +3414,39 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName) } @Override - public WMFullResourcePlan getResourcePlan(String resourcePlanName) + public WMFullResourcePlan getResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException { WMGetResourcePlanRequest request = new WMGetResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); return client.get_resource_plan(request).getResourcePlan(); } @Override - public List getAllResourcePlans() + public List getAllResourcePlans(String ns) throws NoSuchObjectException, MetaException, TException { WMGetAllResourcePlanRequest request = new WMGetAllResourcePlanRequest(); + request.setNs(ns); return client.get_all_resource_plans(request).getResourcePlans(); } @Override - public void dropResourcePlan(String resourcePlanName) + public void dropResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException { WMDropResourcePlanRequest request = new WMDropResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); client.drop_resource_plan(request); } @Override - public WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String resourcePlanName, String ns, + WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) throws NoSuchObjectException, InvalidObjectException, MetaException, TException { WMAlterResourcePlanRequest request = new WMAlterResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); request.setResourcePlan(resourcePlan); request.setIsEnableAndActivate(canActivateDisabled); request.setIsForceDeactivate(isForceDeactivate); @@ -3450,15 +3456,18 @@ public WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableR } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException, TException { - return client.get_active_resource_plan(new WMGetActiveResourcePlanRequest()).getResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException, TException { + WMGetActiveResourcePlanRequest request = new WMGetActiveResourcePlanRequest(); + request.setNs(ns); + return client.get_active_resource_plan(request).getResourcePlan(); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName) + public WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException, TException { WMValidateResourcePlanRequest request = new WMValidateResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); return client.validate_resource_plan(request); } @@ -3479,19 +3488,21 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, MetaException, TException { WMDropTriggerRequest request = new WMDropTriggerRequest(); request.setResourcePlanName(resourcePlanName); request.setTriggerName(triggerName); + request.setNs(ns); client.drop_wm_trigger(request); } @Override - public List getTriggersForResourcePlan(String resourcePlan) + public List getTriggersForResourcePlan(String resourcePlan, String ns) throws NoSuchObjectException, MetaException, TException { WMGetTriggersForResourePlanRequest request = new WMGetTriggersForResourePlanRequest(); request.setResourcePlanName(resourcePlan); + request.setNs(ns); return client.get_triggers_for_resourceplan(request).getTriggers(); } @@ -3513,11 +3524,12 @@ public void alterWMPool(WMNullablePool pool, String poolPath) } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, MetaException, TException { WMDropPoolRequest request = new WMDropPoolRequest(); request.setResourcePlanName(resourcePlanName); request.setPoolPath(poolPath); + request.setNs(ns); client.drop_wm_pool(request); } @@ -3540,13 +3552,14 @@ public void dropWMMapping(WMMapping mapping) @Override public void createOrDropTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath, boolean shouldDrop) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, boolean shouldDrop, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException { WMCreateOrDropTriggerToPoolMappingRequest request = new WMCreateOrDropTriggerToPoolMappingRequest(); request.setResourcePlanName(resourcePlanName); request.setTriggerName(triggerName); request.setPoolPath(poolPath); request.setDrop(shouldDrop); + request.setNs(ns); client.create_or_drop_wm_trigger_to_pool_mapping(request); } diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index d1c0c4d1f60016f28cea69348b1b30ecb61bf083..fa19440ba29446bffb00de1ae0cf5e64fbcd4d12 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -3542,22 +3542,22 @@ void addCheckConstraint(List checkConstraints) throws void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName) throws InvalidObjectException, MetaException, TException; - WMFullResourcePlan getResourcePlan(String resourcePlanName) + WMFullResourcePlan getResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException; - List getAllResourcePlans() + List getAllResourcePlans(String ns) throws NoSuchObjectException, MetaException, TException; - void dropResourcePlan(String resourcePlanName) + void dropResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException; - WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableResourcePlan resourcePlan, + WMFullResourcePlan alterResourcePlan(String resourcePlanName, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) throws NoSuchObjectException, InvalidObjectException, MetaException, TException; - WMFullResourcePlan getActiveResourcePlan() throws MetaException, TException; + WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException, TException; - WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName) + WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException, TException; void createWMTrigger(WMTrigger trigger) @@ -3566,10 +3566,10 @@ void createWMTrigger(WMTrigger trigger) void alterWMTrigger(WMTrigger trigger) throws NoSuchObjectException, InvalidObjectException, MetaException, TException; - void dropWMTrigger(String resourcePlanName, String triggerName) + void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, MetaException, TException; - List getTriggersForResourcePlan(String resourcePlan) + List getTriggersForResourcePlan(String resourcePlan, String ns) throws NoSuchObjectException, MetaException, TException; void createWMPool(WMPool pool) @@ -3578,7 +3578,7 @@ void createWMPool(WMPool pool) void alterWMPool(WMNullablePool pool, String poolPath) throws NoSuchObjectException, InvalidObjectException, TException; - void dropWMPool(String resourcePlanName, String poolPath) + void dropWMPool(String resourcePlanName, String poolPath, String ns) throws TException; void createOrUpdateWMMapping(WMMapping mapping, boolean isUpdate) @@ -3588,7 +3588,7 @@ void dropWMMapping(WMMapping mapping) throws TException; void createOrDropTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath, boolean shouldDrop) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, boolean shouldDrop, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException; /** diff --git standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 4b7b61520a2d55635f474317053a17410f3a4bb7..48b633e5fae02db6e6d20f9121126ef147726566 100644 --- standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -1343,6 +1343,7 @@ struct WMResourcePlan { 2: optional WMResourcePlanStatus status; 3: optional i32 queryParallelism; 4: optional string defaultPoolPath; + 5: optional string ns; } struct WMNullableResourcePlan { @@ -1352,6 +1353,7 @@ struct WMNullableResourcePlan { 5: optional bool isSetQueryParallelism; 6: optional string defaultPoolPath; 7: optional bool isSetDefaultPoolPath; + 8: optional string ns; } struct WMPool { @@ -1360,6 +1362,7 @@ struct WMPool { 3: optional double allocFraction; 4: optional i32 queryParallelism; 5: optional string schedulingPolicy; + 6: optional string ns; } @@ -1370,6 +1373,7 @@ struct WMNullablePool { 4: optional i32 queryParallelism; 5: optional string schedulingPolicy; 6: optional bool isSetSchedulingPolicy; + 7: optional string ns; } struct WMTrigger { @@ -1378,6 +1382,7 @@ struct WMTrigger { 3: optional string triggerExpression; 4: optional string actionExpression; 5: optional bool isInUnmanaged; + 6: optional string ns; } struct WMMapping { @@ -1386,11 +1391,13 @@ struct WMMapping { 3: required string entityName; 4: optional string poolPath; 5: optional i32 ordering; + 6: optional string ns; } struct WMPoolTrigger { 1: required string pool; 2: required string trigger; + 3: optional string ns; } struct WMFullResourcePlan { @@ -1412,6 +1419,7 @@ struct WMCreateResourcePlanResponse { } struct WMGetActiveResourcePlanRequest { + 1: optional string ns; } struct WMGetActiveResourcePlanResponse { @@ -1420,6 +1428,7 @@ struct WMGetActiveResourcePlanResponse { struct WMGetResourcePlanRequest { 1: optional string resourcePlanName; + 2: optional string ns; } struct WMGetResourcePlanResponse { @@ -1427,6 +1436,7 @@ struct WMGetResourcePlanResponse { } struct WMGetAllResourcePlanRequest { + 1: optional string ns; } struct WMGetAllResourcePlanResponse { @@ -1439,6 +1449,7 @@ struct WMAlterResourcePlanRequest { 3: optional bool isEnableAndActivate; 4: optional bool isForceDeactivate; 5: optional bool isReplace; + 6: optional string ns; } struct WMAlterResourcePlanResponse { @@ -1447,6 +1458,7 @@ struct WMAlterResourcePlanResponse { struct WMValidateResourcePlanRequest { 1: optional string resourcePlanName; + 2: optional string ns; } struct WMValidateResourcePlanResponse { @@ -1456,6 +1468,7 @@ struct WMValidateResourcePlanResponse { struct WMDropResourcePlanRequest { 1: optional string resourcePlanName; + 2: optional string ns; } struct WMDropResourcePlanResponse { @@ -1478,6 +1491,7 @@ struct WMAlterTriggerResponse { struct WMDropTriggerRequest { 1: optional string resourcePlanName; 2: optional string triggerName; + 3: optional string ns; } struct WMDropTriggerResponse { @@ -1485,6 +1499,7 @@ struct WMDropTriggerResponse { struct WMGetTriggersForResourePlanRequest { 1: optional string resourcePlanName; + 2: optional string ns; } struct WMGetTriggersForResourePlanResponse { @@ -1509,6 +1524,7 @@ struct WMAlterPoolResponse { struct WMDropPoolRequest { 1: optional string resourcePlanName; 2: optional string poolPath; + 3: optional string ns; } struct WMDropPoolResponse { @@ -1534,6 +1550,7 @@ struct WMCreateOrDropTriggerToPoolMappingRequest { 2: optional string triggerName; 3: optional string poolPath; 4: optional bool drop; + 5: optional string ns; } struct WMCreateOrDropTriggerToPoolMappingResponse { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 8cd46e3f44e7c4e47fbf7f2ce2b6350a5814106f..04851845543d326407a04d33af611e00c9f2c76f 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -8200,7 +8200,7 @@ public WMCreateResourcePlanResponse create_resource_plan(WMCreateResourcePlanReq public WMGetResourcePlanResponse get_resource_plan(WMGetResourcePlanRequest request) throws NoSuchObjectException, MetaException, TException { try { - WMFullResourcePlan rp = getMS().getResourcePlan(request.getResourcePlanName()); + WMFullResourcePlan rp = getMS().getResourcePlan(request.getResourcePlanName(), request.getNs()); WMGetResourcePlanResponse resp = new WMGetResourcePlanResponse(); resp.setResourcePlan(rp); return resp; @@ -8215,7 +8215,7 @@ public WMGetAllResourcePlanResponse get_all_resource_plans(WMGetAllResourcePlanR throws MetaException, TException { try { WMGetAllResourcePlanResponse resp = new WMGetAllResourcePlanResponse(); - resp.setResourcePlans(getMS().getAllResourcePlans()); + resp.setResourcePlans(getMS().getAllResourcePlans(request.getNs())); return resp; } catch (MetaException e) { LOG.error("Exception while trying to retrieve resource plans", e); @@ -8235,7 +8235,7 @@ public WMAlterResourcePlanResponse alter_resource_plan(WMAlterResourcePlanReques // This method will only return full resource plan when activating one, // to give the caller the result atomically with the activation. WMFullResourcePlan fullPlanAfterAlter = getMS().alterResourcePlan( - request.getResourcePlanName(), request.getResourcePlan(), + request.getResourcePlanName(), request.getNs(), request.getResourcePlan(), request.isIsEnableAndActivate(), request.isIsForceDeactivate(), request.isIsReplace()); if (fullPlanAfterAlter != null) { response.setFullResourcePlan(fullPlanAfterAlter); @@ -8252,7 +8252,7 @@ public WMGetActiveResourcePlanResponse get_active_resource_plan( WMGetActiveResourcePlanRequest request) throws MetaException, TException { try { WMGetActiveResourcePlanResponse response = new WMGetActiveResourcePlanResponse(); - response.setResourcePlan(getMS().getActiveResourcePlan()); + response.setResourcePlan(getMS().getActiveResourcePlan(request.getNs())); return response; } catch (MetaException e) { LOG.error("Exception while trying to get active resource plan", e); @@ -8264,7 +8264,7 @@ public WMGetActiveResourcePlanResponse get_active_resource_plan( public WMValidateResourcePlanResponse validate_resource_plan(WMValidateResourcePlanRequest request) throws NoSuchObjectException, MetaException, TException { try { - return getMS().validateResourcePlan(request.getResourcePlanName()); + return getMS().validateResourcePlan(request.getResourcePlanName(), request.getNs()); } catch (MetaException e) { LOG.error("Exception while trying to validate resource plan", e); throw e; @@ -8275,7 +8275,7 @@ public WMValidateResourcePlanResponse validate_resource_plan(WMValidateResourceP public WMDropResourcePlanResponse drop_resource_plan(WMDropResourcePlanRequest request) throws NoSuchObjectException, InvalidOperationException, MetaException, TException { try { - getMS().dropResourcePlan(request.getResourcePlanName()); + getMS().dropResourcePlan(request.getResourcePlanName(), request.getNs()); return new WMDropResourcePlanResponse(); } catch (MetaException e) { LOG.error("Exception while trying to drop resource plan", e); @@ -8311,7 +8311,7 @@ public WMAlterTriggerResponse alter_wm_trigger(WMAlterTriggerRequest request) public WMDropTriggerResponse drop_wm_trigger(WMDropTriggerRequest request) throws NoSuchObjectException, InvalidOperationException, MetaException, TException { try { - getMS().dropWMTrigger(request.getResourcePlanName(), request.getTriggerName()); + getMS().dropWMTrigger(request.getResourcePlanName(), request.getTriggerName(), request.getNs()); return new WMDropTriggerResponse(); } catch (MetaException e) { LOG.error("Exception while trying to drop trigger.", e); @@ -8325,7 +8325,7 @@ public WMGetTriggersForResourePlanResponse get_triggers_for_resourceplan( throws NoSuchObjectException, MetaException, TException { try { List triggers = - getMS().getTriggersForResourcePlan(request.getResourcePlanName()); + getMS().getTriggersForResourcePlan(request.getResourcePlanName(), request.getNs()); WMGetTriggersForResourePlanResponse response = new WMGetTriggersForResourePlanResponse(); response.setTriggers(triggers); return response; @@ -8365,7 +8365,7 @@ public WMCreatePoolResponse create_wm_pool(WMCreatePoolRequest request) public WMDropPoolResponse drop_wm_pool(WMDropPoolRequest request) throws NoSuchObjectException, InvalidOperationException, MetaException, TException { try { - getMS().dropWMPool(request.getResourcePlanName(), request.getPoolPath()); + getMS().dropWMPool(request.getResourcePlanName(), request.getPoolPath(), request.getNs()); return new WMDropPoolResponse(); } catch (MetaException e) { LOG.error("Exception while trying to drop WMPool", e); @@ -8404,11 +8404,11 @@ public WMCreateOrDropTriggerToPoolMappingResponse create_or_drop_wm_trigger_to_p NoSuchObjectException, InvalidObjectException, MetaException, TException { try { if (request.isDrop()) { - getMS().dropWMTriggerToPoolMapping( - request.getResourcePlanName(), request.getTriggerName(), request.getPoolPath()); + getMS().dropWMTriggerToPoolMapping(request.getResourcePlanName(), + request.getTriggerName(), request.getPoolPath(), request.getNs()); } else { - getMS().createWMTriggerToPoolMapping( - request.getResourcePlanName(), request.getTriggerName(), request.getPoolPath()); + getMS().createWMTriggerToPoolMapping(request.getResourcePlanName(), + request.getTriggerName(), request.getPoolPath(), request.getNs()); } return new WMCreateOrDropTriggerToPoolMappingResponse(); } catch (MetaException e) { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 9c158040497cd3d2762620ce35e2b46bb6d5fffe..37e6223b37abc444027bd0aa482940697b150e09 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -11593,13 +11593,15 @@ public void createResourcePlan( } else { rp = new MWMResourcePlan(rpName, null, Status.DISABLED); } + rp.setNs(resourcePlan.getNs()); try { openTransaction(); pm.makePersistent(rp); if (copyFromName != null) { - MWMResourcePlan copyFrom = getMWMResourcePlan(copyFromName, false); + String ns = getNsOrDefault(resourcePlan.getNs()); + MWMResourcePlan copyFrom = getMWMResourcePlan(copyFromName, ns, false); if (copyFrom == null) { - throw new NoSuchObjectException(copyFromName); + throw new NoSuchObjectException(copyFromName + " in " + ns); } copyRpContents(rp, copyFrom); } else { @@ -11627,6 +11629,7 @@ public void createResourcePlan( private void copyRpContents(MWMResourcePlan dest, MWMResourcePlan src) { dest.setQueryParallelism(src.getQueryParallelism()); + dest.setNs(src.getNs()); Map pools = new HashMap<>(); Map> triggersToPools = new HashMap<>(); for (MWMPool copyPool : src.getPools()) { @@ -11686,6 +11689,7 @@ private WMResourcePlan fromMResourcePlan(MWMResourcePlan mplan) { } WMResourcePlan rp = new WMResourcePlan(); rp.setName(mplan.getName()); + rp.setNs(mplan.getNs()); rp.setStatus(WMResourcePlanStatus.valueOf(mplan.getStatus().name())); if (mplan.getQueryParallelism() != null) { rp.setQueryParallelism(mplan.getQueryParallelism()); @@ -11724,6 +11728,7 @@ private WMPool fromMPool(MWMPool mPool, String rpName) { assert mPool.getQueryParallelism() != null; result.setQueryParallelism(mPool.getQueryParallelism()); result.setSchedulingPolicy(mPool.getSchedulingPolicy()); + result.setNs(mPool.getResourcePlan().getNs()); return result; } @@ -11736,15 +11741,24 @@ private WMMapping fromMMapping(MWMMapping mMapping, String rpName) { if (mMapping.getOrdering() != null) { result.setOrdering(mMapping.getOrdering()); } + result.setNs(mMapping.getResourcePlan().getNs()); return result; } + private final String getNsOrDefault(String ns) { + // This is only needed for old clients not setting NS in requests. + // Not clear how to handle this... this is properly a HS2 config but metastore needs its default + // value for backward compat, and we don't want it configurable separately because it's also + // used in upgrade scripts, were it cannot be configured. + return normalizeIdentifier(ns == null ? "default" : ns); + } + @Override - public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException { + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException { boolean commited = false; try { openTransaction(); - WMFullResourcePlan fullRp = fullFromMResourcePlan(getMWMResourcePlan(name, false)); + WMFullResourcePlan fullRp = fullFromMResourcePlan(getMWMResourcePlan(name, ns, false)); commited = commitTransaction(); return fullRp; } catch (InvalidOperationException e) { @@ -11755,12 +11769,12 @@ public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectExcept } } - private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck) + private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck) throws NoSuchObjectException, InvalidOperationException { - return getMWMResourcePlan(name, editCheck, true); + return getMWMResourcePlan(name, ns, editCheck, true); } - private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck, boolean mustExist) + private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck, boolean mustExist) throws NoSuchObjectException, InvalidOperationException { MWMResourcePlan resourcePlan; boolean commited = false; @@ -11768,18 +11782,16 @@ private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck, boole name = normalizeIdentifier(name); try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "name == rpname"); - query.declareParameters("java.lang.String rpname"); - query.setUnique(true); - resourcePlan = (MWMResourcePlan) query.execute(name); + query = createGetResourcePlanQuery(); + ns = getNsOrDefault(ns); + resourcePlan = (MWMResourcePlan) query.execute(name, ns); pm.retrieve(resourcePlan); commited = commitTransaction(); } finally { rollbackAndCleanup(commited, query); } if (mustExist && resourcePlan == null) { - throw new NoSuchObjectException("There is no resource plan named: " + name); + throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); } if (editCheck && resourcePlan != null && resourcePlan.getStatus() != MWMResourcePlan.Status.DISABLED) { @@ -11788,15 +11800,24 @@ private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck, boole return resourcePlan; } + private Query createGetResourcePlanQuery() { + openTransaction(); + Query query = pm.newQuery(MWMResourcePlan.class, "name == rpname && ns == nsname"); + query.declareParameters("java.lang.String rpname, java.lang.String nsname"); + query.setUnique(true); + return query; + } + @Override - public List getAllResourcePlans() throws MetaException { + public List getAllResourcePlans(String ns) throws MetaException { List resourcePlans = new ArrayList(); boolean commited = false; Query query = null; try { openTransaction(); - query = pm.newQuery(MWMResourcePlan.class); - List mplans = (List) query.execute(); + query = pm.newQuery(MWMResourcePlan.class, "ns == nsname"); + query.declareParameters("java.lang.String nsname"); + List mplans = (List) query.execute(getNsOrDefault(ns)); pm.retrieveAll(mplans); commited = commitTransaction(); if (mplans != null) { @@ -11811,7 +11832,7 @@ private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck, boole } @Override - public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan changes, + public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan changes, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { name = name == null ? null : normalizeIdentifier(name); @@ -11827,9 +11848,9 @@ public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan try { openTransaction(); if (isReplace) { - result = handleAlterReplace(name, changes); + result = handleAlterReplace(name, ns, changes); } else { - result = handleSimpleAlter(name, changes, canActivateDisabled, canDeactivate); + result = handleSimpleAlter(name, ns, changes, canActivateDisabled, canDeactivate); } commited = commitTransaction(); @@ -11842,11 +11863,16 @@ public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan } } - private WMFullResourcePlan handleSimpleAlter(String name, WMNullableResourcePlan changes, + private WMFullResourcePlan handleSimpleAlter(String name, String ns, WMNullableResourcePlan changes, boolean canActivateDisabled, boolean canDeactivate) throws InvalidOperationException, NoSuchObjectException, MetaException { - MWMResourcePlan plan = name == null ? getActiveMWMResourcePlan() - : getMWMResourcePlan(name, !changes.isSetStatus()); + MWMResourcePlan plan = name == null ? getActiveMWMResourcePlan(ns) + : getMWMResourcePlan(name, ns, !changes.isSetStatus()); + boolean hasNsChange = changes.isSetNs() && !changes.getNs().equals(getNsOrDefault(plan.getNs())); + if (hasNsChange) { + throw new InvalidOperationException("Cannot change ns; from " + getNsOrDefault(plan.getNs()) + + " to " + changes.getNs()); + } boolean hasNameChange = changes.isSetName() && !changes.getName().equals(name); // Verify that field changes are consistent with what Hive does. Note: we could handle this. if (changes.isSetIsSetQueryParallelism() @@ -11895,7 +11921,7 @@ private WMFullResourcePlan handleSimpleAlter(String name, WMNullableResourcePlan return null; } - private WMFullResourcePlan handleAlterReplace(String name, WMNullableResourcePlan changes) + private WMFullResourcePlan handleAlterReplace(String name, String ns, WMNullableResourcePlan changes) throws InvalidOperationException, NoSuchObjectException, MetaException { // Verify that field changes are consistent with what Hive does. Note: we could handle this. if (changes.isSetQueryParallelism() || changes.isSetDefaultPoolPath()) { @@ -11911,19 +11937,26 @@ private WMFullResourcePlan handleAlterReplace(String name, WMNullableResourcePla if (name == null) { throw new InvalidOperationException("Invalid replace - no name specified"); } + ns = getNsOrDefault(ns); MWMResourcePlan replacedPlan = isReplacingSpecific - ? getMWMResourcePlan(changes.getName(), false) : getActiveMWMResourcePlan(); - MWMResourcePlan plan = getMWMResourcePlan(name, false); + ? getMWMResourcePlan(changes.getName(), ns, false) : getActiveMWMResourcePlan(ns); + MWMResourcePlan plan = getMWMResourcePlan(name, ns, false); if (replacedPlan.getName().equals(plan.getName())) { throw new InvalidOperationException("A plan cannot replace itself"); } + String oldNs = getNsOrDefault(replacedPlan.getNs()), newNs = getNsOrDefault(plan.getNs()); + if (!oldNs.equals(newNs)) { + throw new InvalidOperationException("Cannot change the namespace; replacing " + + oldNs + " with " + newNs); + } + // We will inherit the name and status from the plan we are replacing. String newName = replacedPlan.getName(); int i = 0; String copyName = generateOldPlanName(newName, i); while (true) { - MWMResourcePlan dup = getMWMResourcePlan(copyName, false, false); + MWMResourcePlan dup = getMWMResourcePlan(copyName, ns, false, false); if (dup == null) { break; } @@ -11950,18 +11983,16 @@ private String generateOldPlanName(String newName, int i) { } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { // Note: fullFromMResroucePlan needs to be called inside the txn, otherwise we could have // deduplicated this with getActiveMWMResourcePlan. boolean commited = false; Query query = null; WMFullResourcePlan result = null; try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "status == activeStatus"); - query.declareParameters("java.lang.String activeStatus"); - query.setUnique(true); - MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(Status.ACTIVE.toString()); + query = createActivePlanQuery(); + MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute( + Status.ACTIVE.toString(), getNsOrDefault(ns)); if (mResourcePlan != null) { result = fullFromMResourcePlan(mResourcePlan); } @@ -11972,16 +12003,14 @@ public WMFullResourcePlan getActiveResourcePlan() throws MetaException { return result; } - private MWMResourcePlan getActiveMWMResourcePlan() throws MetaException { + private MWMResourcePlan getActiveMWMResourcePlan(String ns) throws MetaException { boolean commited = false; Query query = null; MWMResourcePlan result = null; try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "status == activeStatus"); - query.declareParameters("java.lang.String activeStatus"); - query.setUnique(true); - result = (MWMResourcePlan) query.execute(Status.ACTIVE.toString()); + query = createActivePlanQuery(); + result = (MWMResourcePlan) query.execute( + Status.ACTIVE.toString(), getNsOrDefault(ns)); pm.retrieve(result); commited = commitTransaction(); } finally { @@ -11990,6 +12019,14 @@ private MWMResourcePlan getActiveMWMResourcePlan() throws MetaException { return result; } + private Query createActivePlanQuery() { + openTransaction(); + Query query = pm.newQuery(MWMResourcePlan.class, "status == activeStatus && ns == nsname"); + query.declareParameters("java.lang.String activeStatus, java.lang.String nsname"); + query.setUnique(true); + return query; + } + private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePlan, String status, boolean canActivateDisabled, boolean canDeactivate) throws InvalidOperationException { Status currentStatus = mResourcePlan.getStatus(); @@ -12018,7 +12055,7 @@ private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePl doValidate = true; doActivate = (newStatus == Status.ACTIVE); if (doActivate && !canActivateDisabled) { - throw new InvalidOperationException("Resource plan " +name + throw new InvalidOperationException("Resource plan " + name + " is disabled and should be enabled before activation (or in the same command)"); } break; @@ -12043,7 +12080,7 @@ private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePl } if (doActivate) { // Deactivate currently active resource plan. - deactivateActiveResourcePlan(); + deactivateActiveResourcePlan(mResourcePlan.getNs()); mResourcePlan.setStatus(newStatus); return fullFromMResourcePlan(mResourcePlan); } else { @@ -12052,14 +12089,13 @@ private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePl return null; } - private void deactivateActiveResourcePlan() { + private void deactivateActiveResourcePlan(String ns) { boolean commited = false; Query query = null; try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "status == \"ACTIVE\""); - query.setUnique(true); - MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(); + query = createActivePlanQuery(); + MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute( + Status.ACTIVE.toString(), getNsOrDefault(ns)); // We may not have an active resource plan in the start. if (mResourcePlan != null) { mResourcePlan.setStatus(Status.ENABLED); @@ -12151,19 +12187,16 @@ private WMValidateResourcePlanResponse getResourcePlanErrors(MWMResourcePlan mRe } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { name = normalizeIdentifier(name); Query query = null; try { - query = pm.newQuery(MWMResourcePlan.class, "name == rpName"); - query.declareParameters("java.lang.String rpName"); - query.setUnique(true); - MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(name); + query = createGetResourcePlanQuery(); + MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(name, getNsOrDefault(ns)); if (mResourcePlan == null) { - throw new NoSuchObjectException("Cannot find resourcePlan: " + name); + throw new NoSuchObjectException("Cannot find resourcePlan: " + name + " in " + ns); } - // Validate resource plan. return getResourcePlanErrors(mResourcePlan); } finally { rollbackAndCleanup(true, query); @@ -12171,19 +12204,16 @@ public WMValidateResourcePlanResponse validateResourcePlan(String name) } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { name = normalizeIdentifier(name); boolean commited = false; Query query = null; try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "name == rpname"); - query.declareParameters("java.lang.String rpname"); - query.setUnique(true); - MWMResourcePlan resourcePlan = (MWMResourcePlan) query.execute(name); - pm.retrieve(resourcePlan); + query = createGetResourcePlanQuery(); + MWMResourcePlan resourcePlan = (MWMResourcePlan) query.execute(name, getNsOrDefault(ns)); + pm.retrieve(resourcePlan); // TODO: why do some codepaths call retrieve and some don't? if (resourcePlan == null) { - throw new NoSuchObjectException("There is no resource plan named: " + name); + throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); } if (resourcePlan.getStatus() == Status.ACTIVE) { throw new MetaException("Cannot drop an active resource plan"); @@ -12207,7 +12237,8 @@ public void createWMTrigger(WMTrigger trigger) boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(trigger.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + trigger.getResourcePlanName(), trigger.getNs(), true); MWMTrigger mTrigger = new MWMTrigger(resourcePlan, normalizeIdentifier(trigger.getTriggerName()), trigger.getTriggerExpression(), trigger.getActionExpression(), null, @@ -12229,7 +12260,8 @@ public void alterWMTrigger(WMTrigger trigger) Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(trigger.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + trigger.getResourcePlanName(), trigger.getNs(), true); MWMTrigger mTrigger = getTrigger(resourcePlan, trigger.getTriggerName()); // Update the object. if (trigger.isSetTriggerExpression()) { @@ -12271,7 +12303,7 @@ private MWMTrigger getTrigger(MWMResourcePlan resourcePlan, String triggerName) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { resourcePlanName = normalizeIdentifier(resourcePlanName); triggerName = normalizeIdentifier(triggerName); @@ -12280,7 +12312,7 @@ public void dropWMTrigger(String resourcePlanName, String triggerName) Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, true); + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp && name == triggerName"); query.declareParameters("MWMResourcePlan rp, java.lang.String triggerName"); if (query.deletePersistentAll(resourcePlan, triggerName) != 1) { @@ -12293,7 +12325,7 @@ public void dropWMTrigger(String resourcePlanName, String triggerName) } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { List triggers = new ArrayList(); boolean commited = false; @@ -12302,7 +12334,7 @@ public void dropWMTrigger(String resourcePlanName, String triggerName) openTransaction(); MWMResourcePlan resourcePlan; try { - resourcePlan = getMWMResourcePlan(resourcePlanName, false); + resourcePlan = getMWMResourcePlan(resourcePlanName, ns, false); } catch (InvalidOperationException e) { // Should not happen, edit check is false. throw new RuntimeException(e); @@ -12330,6 +12362,7 @@ private WMTrigger fromMWMTrigger(MWMTrigger mTrigger, String resourcePlanName) { trigger.setTriggerExpression(mTrigger.getTriggerExpression()); trigger.setActionExpression(mTrigger.getActionExpression()); trigger.setIsInUnmanaged(mTrigger.getIsInUnmanaged()); + trigger.setNs(mTrigger.getResourcePlan().getNs()); return trigger; } @@ -12339,7 +12372,8 @@ public void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectE boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(pool.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + pool.getResourcePlanName(), pool.getNs(), true); if (!poolParentExists(resourcePlan, pool.getPoolPath())) { throw new NoSuchObjectException("Pool path is invalid, the parent does not exist"); @@ -12366,7 +12400,8 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(pool.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + pool.getResourcePlanName(), pool.getNs(), true); MWMPool mPool = getPool(resourcePlan, poolPath); pm.retrieve(mPool); if (pool.isSetAllocFraction()) { @@ -12463,20 +12498,20 @@ private boolean poolParentExists(MWMResourcePlan resourcePlan, String poolPath) } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { poolPath = normalizeIdentifier(poolPath); boolean commited = false; Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, true); + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); if (resourcePlan.getDefaultPool() != null && resourcePlan.getDefaultPool().getPath().equals(poolPath)) { throw new InvalidOperationException("Cannot drop default pool of a resource plan"); } if (poolHasChildren(resourcePlan, poolPath)) { - throw new InvalidOperationException("Pool has children cannot drop."); + throw new InvalidOperationException("Cannot drop a pool that has child pools"); } query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)"); query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); @@ -12521,7 +12556,8 @@ public void createOrUpdateWMMapping(WMMapping mapping, boolean update) Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(mapping.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + mapping.getResourcePlanName(), mapping.getNs(), true); MWMPool pool = null; if (mapping.isSetPoolPath()) { pool = getPool(resourcePlan, mapping.getPoolPath()); @@ -12555,7 +12591,8 @@ public void dropWMMapping(WMMapping mapping) Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(mapping.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + mapping.getResourcePlanName(), mapping.getNs(), true); query = pm.newQuery(MWMMapping.class, "resourcePlan == rp && entityType == type && entityName == name"); query.declareParameters("MWMResourcePlan rp, java.lang.String type, java.lang.String name"); @@ -12570,12 +12607,12 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, true); + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); MWMPool pool = getPool(resourcePlan, poolPath); MWMTrigger trigger = getTrigger(resourcePlan, triggerName); pool.getTriggers().add(trigger); @@ -12590,11 +12627,11 @@ public void createWMTriggerToPoolMapping(String resourcePlanName, String trigger @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, true); + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); MWMPool pool = getPool(resourcePlan, poolPath); MWMTrigger trigger = getTrigger(resourcePlan, triggerName); pool.getTriggers().remove(trigger); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index c3914b668fac18ead6196a4fc449e909f5af01b1..209e92a415c7698c6ce4c4b5a3884aa61b8bb833 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -20,12 +20,6 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.*; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; -import org.apache.hadoop.hive.metastore.api.ISchemaName; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; -import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; -import org.apache.hadoop.hive.metastore.api.WriteEventInfo; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -1546,21 +1540,21 @@ void dropConstraint(String catName, String dbName, String tableName, String cons void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int defaultPoolSize) throws AlreadyExistsException, MetaException, InvalidObjectException, NoSuchObjectException; - WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException, MetaException; + WMFullResourcePlan getResourcePlan(String name, String string) throws NoSuchObjectException, MetaException; - List getAllResourcePlans() throws MetaException; + List getAllResourcePlans(String string) throws MetaException; - WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, + WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException; - WMFullResourcePlan getActiveResourcePlan() throws MetaException; + WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException; - WMValidateResourcePlanResponse validateResourcePlan(String name) + WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException; - void dropResourcePlan(String name) throws NoSuchObjectException, MetaException; + void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException; void createWMTrigger(WMTrigger trigger) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, @@ -1569,10 +1563,10 @@ void createWMTrigger(WMTrigger trigger) void alterWMTrigger(WMTrigger trigger) throws NoSuchObjectException, InvalidOperationException, MetaException; - void dropWMTrigger(String resourcePlanName, String triggerName) + void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException; - List getTriggersForResourcePlan(String resourcePlanName) + List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException; void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectException, @@ -1581,7 +1575,7 @@ void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectExceptio void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException; - void dropWMPool(String resourcePlanName, String poolPath) + void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException; void createOrUpdateWMMapping(WMMapping mapping, boolean update) @@ -1591,11 +1585,11 @@ void createOrUpdateWMMapping(WMMapping mapping, boolean update) void dropWMMapping(WMMapping mapping) throws NoSuchObjectException, InvalidOperationException, MetaException; - void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath) + void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException; - void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath) + void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException; /** diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 47ac68c667bea8f09f5301a6364c854bc18b3c0d..e4ef46fdb4be4dea25bc8e0839e159413223fd35 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -1644,7 +1644,7 @@ public Partition getPartitionWithAuth(String catName, String dbName, String tblN } return partitions; } - + private String getPartNameMatcher(Table table, List partSpecs) throws MetaException { List partCols = table.getPartitionKeys(); int numPartKeys = partCols.size(); @@ -2407,39 +2407,39 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int } @Override - public WMFullResourcePlan getResourcePlan(String name) + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { - return rawStore.getResourcePlan(name); + return rawStore.getResourcePlan(name, ns); } @Override - public List getAllResourcePlans() throws MetaException { - return rawStore.getAllResourcePlans(); + public List getAllResourcePlans(String ns) throws MetaException { + return rawStore.getAllResourcePlans(ns); } @Override - public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { return rawStore.alterResourcePlan( - name, resourcePlan, canActivateDisabled, canDeactivate, isReplace); + name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { - return rawStore.getActiveResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { + return rawStore.getActiveResourcePlan(ns); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { - return rawStore.validateResourcePlan(name); + return rawStore.validateResourcePlan(name, ns); } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { - rawStore.dropResourcePlan(name); + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + rawStore.dropResourcePlan(name, ns); } @Override @@ -2456,15 +2456,15 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMTrigger(resourcePlanName, triggerName); + rawStore.dropWMTrigger(resourcePlanName, triggerName, ns); } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { - return rawStore.getTriggersForResourcePlan(resourcePlanName); + return rawStore.getTriggersForResourcePlan(resourcePlanName, ns); } @Override @@ -2480,9 +2480,9 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMPool(resourcePlanName, poolPath); + rawStore.dropWMPool(resourcePlanName, poolPath, ns); } @Override @@ -2500,15 +2500,15 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + rawStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { + rawStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } public long getCacheUpdateCount() { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MWMResourcePlan.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MWMResourcePlan.java index ac51f2d77145b37da468ce8df2ac5c42f4d6c538..648587e5bf049bb59c7b4f9a53e686e961044bf3 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MWMResourcePlan.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MWMResourcePlan.java @@ -25,6 +25,7 @@ */ public class MWMResourcePlan { private String name; + private String ns; private Integer queryParallelism; private Status status; private Set pools; @@ -51,6 +52,14 @@ public String getName() { return name; } + public void setNs(String name) { + this.ns = name; + } + + public String getNs() { + return ns; + } + public void setName(String name) { this.name = name; } diff --git standalone-metastore/metastore-server/src/main/resources/package.jdo standalone-metastore/metastore-server/src/main/resources/package.jdo index fef6a42038bb2aa0cba6dfda8d710fd37cb720e7..01306e6fab8d8717e22464e501b1dd04b6c013dc 100644 --- standalone-metastore/metastore-server/src/main/resources/package.jdo +++ standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -1230,6 +1230,9 @@ + + + @@ -1241,6 +1244,7 @@ + diff --git standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql index c889bbdf96b887b29be858e41ee854f0731cd5cd..a3c4196dbff7e53be5317631b314983d16a99020 100644 --- standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -182,7 +182,7 @@ CREATE TABLE "APP"."KEY_CONSTRAINTS" ("CHILD_CD_ID" BIGINT, "CHILD_INTEGER_IDX" CREATE TABLE "APP"."METASTORE_DB_PROPERTIES" ("PROPERTY_KEY" VARCHAR(255) NOT NULL, "PROPERTY_VALUE" VARCHAR(1000) NOT NULL, "DESCRIPTION" VARCHAR(1000)); -CREATE TABLE "APP"."WM_RESOURCEPLAN" (RP_ID BIGINT NOT NULL, NAME VARCHAR(128) NOT NULL, QUERY_PARALLELISM INTEGER, STATUS VARCHAR(20) NOT NULL, DEFAULT_POOL_ID BIGINT); +CREATE TABLE "APP"."WM_RESOURCEPLAN" (RP_ID BIGINT NOT NULL, NS VARCHAR(128), NAME VARCHAR(128) NOT NULL, QUERY_PARALLELISM INTEGER, STATUS VARCHAR(20) NOT NULL, DEFAULT_POOL_ID BIGINT); CREATE TABLE "APP"."WM_POOL" (POOL_ID BIGINT NOT NULL, RP_ID BIGINT NOT NULL, PATH VARCHAR(1024) NOT NULL, ALLOC_FRACTION DOUBLE, QUERY_PARALLELISM INTEGER, SCHEDULING_POLICY VARCHAR(1024)); @@ -264,7 +264,7 @@ CREATE INDEX "APP"."CONSTRAINTS_PARENT_TBL_ID_INDEX" ON "APP"."KEY_CONSTRAINTS"( CREATE INDEX "APP"."CONSTRAINTS_CONSTRAINT_TYPE_INDEX" ON "APP"."KEY_CONSTRAINTS"("CONSTRAINT_TYPE"); -CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN" ("NAME"); +CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN" ("NS", "NAME"); CREATE UNIQUE INDEX "APP"."UNIQUE_WM_POOL" ON "APP"."WM_POOL" ("RP_ID", "PATH"); diff --git standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql index aca5227a5bb6192da6c5f070c04d2941d636bad2..bcaebd18accf86846ae44a6498046514575fc069 100644 --- standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql +++ standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql @@ -3,6 +3,13 @@ ALTER TABLE "APP"."TBLS" ADD WRITE_ID bigint DEFAULT 0; ALTER TABLE "APP"."PARTITIONS" ADD WRITE_ID bigint DEFAULT 0; +-- HIVE-20793 +ALTER TABLE "APP"."WM_RESOURCEPLAN" ADD NS VARCHAR(128); +UPDATE "APP"."WM_RESOURCEPLAN" SET NS = 'default' WHERE NS IS NULL; +DROP INDEX "APP"."UNIQUE_WM_RESOURCEPLAN"; +CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN" ("NS", "NAME"); + + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 91ba134325094e413887a89e1d605efa99218288..5ea1b4450d8258e841bb4af7381ca6fb0ba1a827 100644 --- standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -642,6 +642,7 @@ CREATE TABLE WM_RESOURCEPLAN ( RP_ID bigint NOT NULL, "NAME" nvarchar(128) NOT NULL, + NS nvarchar(128), QUERY_PARALLELISM int, STATUS nvarchar(20) NOT NULL, DEFAULT_POOL_ID bigint @@ -963,7 +964,7 @@ CREATE INDEX TABLE_PARAMS_N49 ON TABLE_PARAMS (TBL_ID); -- Constraints for resource plan tables. -CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NAME"); +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NS", "NAME"); CREATE UNIQUE INDEX UNIQUE_WM_POOL ON WM_POOL (RP_ID, PATH); diff --git standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql index f0d861b3a9bc982c1e24fa49415dcfc6c105cd68..edde08db9ef7ee01800c7cc3a04c813014abdd18 100644 --- standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql +++ standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql @@ -4,6 +4,12 @@ SELECT 'Upgrading MetaStore schema from 3.2.0 to 4.0.0' AS MESSAGE; ALTER TABLE TBLS ADD WRITE_ID bigint NOT NULL DEFAULT 0; ALTER TABLE PARTITIONS ADD WRITE_ID bigint NOT NULL DEFAULT 0; +-- HIVE-20793 +ALTER TABLE WM_RESOURCEPLAN ADD NS nvarchar(128); +UPDATE WM_RESOURCEPLAN SET NS = 'default' WHERE NS IS NULL; +DROP INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN; +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NS", "NAME"); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS MESSAGE; diff --git standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 3af2ebb253f82bb85976d229d4ac2225deffdbde..5555a59c7d7e933d25d8d5af611e5b6aa0c0c19b 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -917,11 +917,12 @@ CREATE TABLE IF NOT EXISTS `METASTORE_DB_PROPERTIES` ( CREATE TABLE IF NOT EXISTS WM_RESOURCEPLAN ( `RP_ID` bigint(20) NOT NULL, `NAME` varchar(128) NOT NULL, + `NS` varchar(128), `QUERY_PARALLELISM` int(11), `STATUS` varchar(20) NOT NULL, `DEFAULT_POOL_ID` bigint(20), PRIMARY KEY (`RP_ID`), - UNIQUE KEY `UNIQUE_WM_RESOURCEPLAN` (`NAME`) + UNIQUE KEY `UNIQUE_WM_RESOURCEPLAN` (`NAME`, `NS`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE TABLE IF NOT EXISTS WM_POOL diff --git standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql index ee0f691b524a6e822ac14e09d24d3a49ae8565b1..701acb00984c61f7511dcc48053890b154575d1f 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql @@ -4,6 +4,13 @@ SELECT 'Upgrading MetaStore schema from 3.2.0 to 4.0.0' AS ' '; ALTER TABLE TBLS ADD WRITE_ID bigint DEFAULT 0; ALTER TABLE PARTITIONS ADD WRITE_ID bigint DEFAULT 0; + +-- HIVE-20793 +ALTER TABLE `WM_RESOURCEPLAN` ADD `NS` varchar(128); +UPDATE `WM_RESOURCEPLAN` SET `NS` = 'default' WHERE `NS` IS NULL; +ALTER TABLE `WM_RESOURCEPLAN` DROP KEY `UNIQUE_WM_RESOURCEPLAN`; +ALTER TABLE `WM_RESOURCEPLAN` ADD UNIQUE KEY `UNIQUE_WM_RESOURCEPLAN` (`NAME`, `NS`); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS ' '; diff --git standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index 33aa08015a9e17585c42d64d44b364be96e69eaf..b1980c5b83f16614845063516495188ebdd8c2a3 100644 --- standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -638,6 +638,7 @@ CREATE TABLE WM_RESOURCEPLAN ( RP_ID NUMBER NOT NULL, "NAME" VARCHAR2(128) NOT NULL, + NS VARCHAR2(128), QUERY_PARALLELISM NUMBER(10), STATUS VARCHAR2(20) NOT NULL, DEFAULT_POOL_ID NUMBER @@ -927,7 +928,7 @@ ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT PROPERTY_KEY_PK PRIMARY KEY ( -- Constraints for resource plan tables. -CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NAME"); +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN (NS, "NAME"); CREATE UNIQUE INDEX UNIQUE_WM_POOL ON WM_POOL (RP_ID, PATH); diff --git standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql index bbb4a39ec4f6f616c9a1a9042a35cafd45cf9796..b9f63313251ab1fa6278b862ed9e07e62b234c04 100644 --- standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql +++ standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql @@ -3,6 +3,12 @@ SELECT 'Upgrading MetaStore schema from 3.2.0 to 4.0.0' AS Status from dual; ALTER TABLE TBLS ADD WRITE_ID number DEFAULT 0 NOT NULL; ALTER TABLE PARTITIONS ADD WRITE_ID number DEFAULT 0 NOT NULL; +-- HIVE-20793 +ALTER TABLE WM_RESOURCEPLAN ADD NS VARCHAR2(128); +UPDATE WM_RESOURCEPLAN SET NS = 'default' WHERE NS IS NULL; +DROP INDEX UNIQUE_WM_RESOURCEPLAN; +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN (NS, "NAME"); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS Status from dual; diff --git standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index ea088d77fdaec85834b8fd3f01eacdfac58dd245..9040005aa82b7a8cc5c01f257ecd47a7cc97e9b2 100644 --- standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -670,6 +670,7 @@ CREATE TABLE "METASTORE_DB_PROPERTIES" CREATE TABLE "WM_RESOURCEPLAN" ( "RP_ID" bigint NOT NULL, "NAME" character varying(128) NOT NULL, + "NS" character varying(128), "QUERY_PARALLELISM" integer, "STATUS" character varying(20) NOT NULL, "DEFAULT_POOL_ID" bigint @@ -1060,7 +1061,7 @@ ALTER TABLE ONLY "WM_RESOURCEPLAN" ADD CONSTRAINT "WM_RESOURCEPLAN_pkey" PRIMARY KEY ("RP_ID"); ALTER TABLE ONLY "WM_RESOURCEPLAN" - ADD CONSTRAINT "UNIQUE_WM_RESOURCEPLAN" UNIQUE ("NAME"); + ADD CONSTRAINT "UNIQUE_WM_RESOURCEPLAN" UNIQUE ("NS", "NAME"); ALTER TABLE ONLY "WM_POOL" ADD CONSTRAINT "WM_POOL_pkey" PRIMARY KEY ("POOL_ID"); diff --git standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql index 2a2d70ae802eb6f5b9ab7f4f9519a0af30d2c5b4..0c36069d071d4b60cc338ba729da5d22e08ca8ca 100644 --- standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql +++ standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql @@ -4,6 +4,13 @@ SELECT 'Upgrading MetaStore schema from 3.2.0 to 4.0.0'; ALTER TABLE "TBLS" ADD "WRITE_ID" bigint DEFAULT 0; ALTER TABLE "PARTITIONS" ADD "WRITE_ID" bigint DEFAULT 0; + +-- HIVE-20793 +ALTER TABLE "WM_RESOURCEPLAN" ADD "NS" character varying(128); +UPDATE "WM_RESOURCEPLAN" SET "NS" = 'default' WHERE "NS" IS NULL; +ALTER TABLE "WM_RESOURCEPLAN" DROP CONSTRAINT "UNIQUE_WM_RESOURCEPLAN"; +ALTER TABLE ONLY "WM_RESOURCEPLAN" ADD CONSTRAINT "UNIQUE_WM_RESOURCEPLAN" UNIQUE ("NS", "NAME"); + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0'; diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 966979891b71f1cbfe50f56c40c35af8b304c47f..c13e538bc4500c9c8197326043ad32ab9f22dca9 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -1060,39 +1060,39 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int } @Override - public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException, + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { - return objectStore.getResourcePlan(name); + return objectStore.getResourcePlan(name, ns); } @Override - public List getAllResourcePlans() throws MetaException { - return objectStore.getAllResourcePlans(); + public List getAllResourcePlans(String ns) throws MetaException { + return objectStore.getAllResourcePlans(ns); } @Override - public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { return objectStore.alterResourcePlan( - name, resourcePlan, canActivateDisabled, canDeactivate, isReplace); + name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { - return objectStore.getActiveResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { + return objectStore.getActiveResourcePlan(ns); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { - return objectStore.validateResourcePlan(name); + return objectStore.validateResourcePlan(name, ns); } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { - objectStore.dropResourcePlan(name); + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + objectStore.dropResourcePlan(name, ns); } @Override @@ -1109,15 +1109,15 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMTrigger(resourcePlanName, triggerName); + objectStore.dropWMTrigger(resourcePlanName, triggerName, ns); } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { - return objectStore.getTriggersForResourcePlan(resourcePlanName); + return objectStore.getTriggersForResourcePlan(resourcePlanName, ns); } @Override @@ -1133,9 +1133,9 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMPool(resourcePlanName, poolPath); + objectStore.dropWMPool(resourcePlanName, poolPath, ns); } @Override @@ -1153,15 +1153,15 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + objectStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { + objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 593d562c3498660861201f58d83c27d59d184046..e943f17a365e847091454e9e37d536c712fb386f 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -1051,36 +1051,36 @@ public void createResourcePlan( } @Override - public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException { + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException { return null; } @Override - public List getAllResourcePlans() throws MetaException { + public List getAllResourcePlans(String ns) throws MetaException { return null; } @Override public WMFullResourcePlan alterResourcePlan( - String name, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, + String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws NoSuchObjectException, InvalidOperationException, MetaException { return null; } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { return null; } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { return null; } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { } @Override @@ -1093,12 +1093,12 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, MetaException { } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { return null; } @@ -1114,7 +1114,7 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { } @@ -1131,13 +1131,13 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { } @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { } @Override diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 4293579ad8b55d59f2230040f23e9a693d838ca7..9fe9a65677f2638c13b31cdfe468ed08979e30aa 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -51,6 +51,7 @@ import javax.security.auth.login.LoginException; import com.google.common.base.Preconditions; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -2833,34 +2834,39 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName) } @Override - public WMFullResourcePlan getResourcePlan(String resourcePlanName) + public WMFullResourcePlan getResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException { WMGetResourcePlanRequest request = new WMGetResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); return client.get_resource_plan(request).getResourcePlan(); } @Override - public List getAllResourcePlans() + public List getAllResourcePlans(String ns) throws NoSuchObjectException, MetaException, TException { WMGetAllResourcePlanRequest request = new WMGetAllResourcePlanRequest(); + request.setNs(ns); return client.get_all_resource_plans(request).getResourcePlans(); } @Override - public void dropResourcePlan(String resourcePlanName) + public void dropResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException { WMDropResourcePlanRequest request = new WMDropResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); client.drop_resource_plan(request); } @Override - public WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String resourcePlanName, String ns, + WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) throws NoSuchObjectException, InvalidObjectException, MetaException, TException { WMAlterResourcePlanRequest request = new WMAlterResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); request.setResourcePlan(resourcePlan); request.setIsEnableAndActivate(canActivateDisabled); request.setIsForceDeactivate(isForceDeactivate); @@ -2870,15 +2876,18 @@ public WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableR } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException, TException { - return client.get_active_resource_plan(new WMGetActiveResourcePlanRequest()).getResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException, TException { + WMGetActiveResourcePlanRequest request = new WMGetActiveResourcePlanRequest(); + request.setNs(ns); + return client.get_active_resource_plan(request).getResourcePlan(); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName) + public WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException, TException { WMValidateResourcePlanRequest request = new WMValidateResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); return client.validate_resource_plan(request); } @@ -2899,19 +2908,21 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, MetaException, TException { WMDropTriggerRequest request = new WMDropTriggerRequest(); request.setResourcePlanName(resourcePlanName); request.setTriggerName(triggerName); + request.setNs(ns); client.drop_wm_trigger(request); } @Override - public List getTriggersForResourcePlan(String resourcePlan) + public List getTriggersForResourcePlan(String resourcePlan, String ns) throws NoSuchObjectException, MetaException, TException { WMGetTriggersForResourePlanRequest request = new WMGetTriggersForResourePlanRequest(); request.setResourcePlanName(resourcePlan); + request.setNs(ns); return client.get_triggers_for_resourceplan(request).getTriggers(); } @@ -2933,11 +2944,12 @@ public void alterWMPool(WMNullablePool pool, String poolPath) } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, MetaException, TException { WMDropPoolRequest request = new WMDropPoolRequest(); request.setResourcePlanName(resourcePlanName); request.setPoolPath(poolPath); + request.setNs(ns); client.drop_wm_pool(request); } @@ -2960,16 +2972,18 @@ public void dropWMMapping(WMMapping mapping) @Override public void createOrDropTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath, boolean shouldDrop) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, boolean shouldDrop, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException { WMCreateOrDropTriggerToPoolMappingRequest request = new WMCreateOrDropTriggerToPoolMappingRequest(); request.setResourcePlanName(resourcePlanName); request.setTriggerName(triggerName); request.setPoolPath(poolPath); request.setDrop(shouldDrop); + request.setNs(ns); client.create_or_drop_wm_trigger_to_pool_mapping(request); } + @Override public void createCatalog(Catalog catalog) throws AlreadyExistsException, InvalidObjectException, MetaException, TException {