diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e226a1f82d44550f389308f91d578e7aa4ea170a..d021310399322df16fb8a0fd0b5af416d1fee259 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3206,6 +3206,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_SERVER2_TEZ_INTERACTIVE_QUEUE("hive.server2.tez.interactive.queue", "", "A single YARN queues to use for Hive Interactive sessions. When this is specified,\n" + "workload management is enabled and used for these sessions."), + HIVE_SERVER2_WM_NAMESPACE("hive.server2.wm.namespace", "default", + "The WM namespace to use when one metastore is used by multiple compute clusters each \n" + + "with their own workload management. The special value 'default' (the default) will \n" + + "also include any resource plans created before the namespaces were introduced."), HIVE_SERVER2_WM_WORKER_THREADS("hive.server2.wm.worker.threads", 4, "Number of worker threads to use to perform the synchronous operations with Tez\n" + "sessions for workload management (e.g. opening, closing, etc.)"), diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index c3e1e8e88c69d7713e16c7061ce8cf73a0d5e833..d9fb645858887feba44e49e7cc91d98327b4f3c1 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -1105,36 +1105,36 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int } @Override - public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException, MetaException { - return objectStore.getResourcePlan(name); + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + return objectStore.getResourcePlan(name, ns); } @Override - public List getAllResourcePlans() throws MetaException { - return objectStore.getAllResourcePlans(); + public List getAllResourcePlans(String ns) throws MetaException { + return objectStore.getAllResourcePlans(ns); } @Override - public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - return objectStore.alterResourcePlan(name, resourcePlan, canActivateDisabled, canDeactivate, isReplace); + return objectStore.alterResourcePlan(name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { - return objectStore.getActiveResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { + return objectStore.getActiveResourcePlan(ns); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { - return objectStore.validateResourcePlan(name); + return objectStore.validateResourcePlan(name, ns); } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { - objectStore.dropResourcePlan(name); + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + objectStore.dropResourcePlan(name, ns); } @Override @@ -1151,15 +1151,15 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMTrigger(resourcePlanName, triggerName); + objectStore.dropWMTrigger(resourcePlanName, triggerName, ns); } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { - return objectStore.getTriggersForResourcePlan(resourcePlanName); + return objectStore.getTriggersForResourcePlan(resourcePlanName, ns); } @Override @@ -1175,9 +1175,9 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMPool(resourcePlanName, poolPath); + objectStore.dropWMPool(resourcePlanName, poolPath, ns); } @Override @@ -1195,15 +1195,15 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + objectStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { + objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } @Override diff --git metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql index a69046f961cdf0fff7989492c489bb62f2a66d72..db1384b639f4f0024b5c83dab7f883c49904bde8 100644 --- metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql +++ metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql @@ -941,6 +941,7 @@ FROM `PARTITION_PARAMS` GROUP BY `PART_ID`; CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( `NAME` string, + `NS` string, `STATUS` string, `QUERY_PARALLELISM` int, `DEFAULT_POOL_PATH` string @@ -951,6 +952,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"STATUS\", \"WM_RESOURCEPLAN\".\"QUERY_PARALLELISM\", \"WM_POOL\".\"PATH\" @@ -960,6 +962,7 @@ FROM CREATE EXTERNAL TABLE IF NOT EXISTS `WM_TRIGGERS` ( `RP_NAME` string, + `NS` string, `NAME` string, `TRIGGER_EXPRESSION` string, `ACTION_EXPRESSION` string @@ -970,6 +973,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT r.\"NAME\" AS RP_NAME, + case when r.\"NS\" is null then 'default' else r.\"NS\" end, t.\"NAME\" AS NAME, \"TRIGGER_EXPRESSION\", \"ACTION_EXPRESSION\" @@ -983,6 +987,7 @@ ON CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS` ( `RP_NAME` string, + `NS` string, `PATH` string, `ALLOC_FRACTION` double, `QUERY_PARALLELISM` int, @@ -994,6 +999,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"WM_POOL\".\"PATH\", \"WM_POOL\".\"ALLOC_FRACTION\", \"WM_POOL\".\"QUERY_PARALLELISM\", @@ -1008,6 +1014,7 @@ ON CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( `RP_NAME` string, + `NS` string, `POOL_PATH` string, `TRIGGER_NAME` string ) @@ -1017,6 +1024,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"WM_POOL\".\"PATH\" AS POOL_PATH, \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME FROM \"WM_POOL_TO_TRIGGER\" @@ -1026,6 +1034,7 @@ FROM \"WM_POOL_TO_TRIGGER\" UNION SELECT \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, '' AS POOL_PATH, \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME FROM \"WM_TRIGGER\" @@ -1036,6 +1045,7 @@ WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't') CREATE EXTERNAL TABLE IF NOT EXISTS `WM_MAPPINGS` ( `RP_NAME` string, + `NS` string, `ENTITY_TYPE` string, `ENTITY_NAME` string, `POOL_PATH` string, @@ -1047,6 +1057,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"ENTITY_TYPE\", \"ENTITY_NAME\", case when \"WM_POOL\".\"PATH\" is null then '' else \"WM_POOL\".\"PATH\" end, diff --git metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql index 4c770206fe3dcceb8570be1c1ef078b376f5cafd..6cb12f912365d379ee7b4392aac726b9922d2b93 100644 --- metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql +++ metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql @@ -2,6 +2,143 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0'; USE SYS; +-- HIVE-20793 +DROP TABLE IF EXISTS `WM_RESOURCEPLANS`; +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( + `NAME` string, + `NS` string, + `STATUS` string, + `QUERY_PARALLELISM` int, + `DEFAULT_POOL_PATH` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, + \"STATUS\", + \"WM_RESOURCEPLAN\".\"QUERY_PARALLELISM\", + \"WM_POOL\".\"PATH\" +FROM + \"WM_RESOURCEPLAN\" LEFT OUTER JOIN \"WM_POOL\" ON \"WM_RESOURCEPLAN\".\"DEFAULT_POOL_ID\" = \"WM_POOL\".\"POOL_ID\"" +); + +DROP TABLE IF EXISTS `WM_TRIGGERS`; +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_TRIGGERS` ( + `RP_NAME` string, + `NS` string, + `NAME` string, + `TRIGGER_EXPRESSION` string, + `ACTION_EXPRESSION` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + r.\"NAME\" AS RP_NAME, + case when r.\"NS\" is null then 'default' else r.\"NS\" end, + t.\"NAME\" AS NAME, + \"TRIGGER_EXPRESSION\", + \"ACTION_EXPRESSION\" +FROM + \"WM_TRIGGER\" t +JOIN + \"WM_RESOURCEPLAN\" r +ON + t.\"RP_ID\" = r.\"RP_ID\"" +); + +DROP TABLE IF EXISTS `WM_POOLS`; +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS` ( + `RP_NAME` string, + `NS` string, + `PATH` string, + `ALLOC_FRACTION` double, + `QUERY_PARALLELISM` int, + `SCHEDULING_POLICY` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, + \"WM_POOL\".\"PATH\", + \"WM_POOL\".\"ALLOC_FRACTION\", + \"WM_POOL\".\"QUERY_PARALLELISM\", + \"WM_POOL\".\"SCHEDULING_POLICY\" +FROM + \"WM_POOL\" +JOIN + \"WM_RESOURCEPLAN\" +ON + \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"" +); + +DROP TABLE IF EXISTS `WM_POOLS_TO_TRIGGERS`; +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( + `RP_NAME` string, + `NS` string, + `POOL_PATH` string, + `TRIGGER_NAME` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, + \"WM_POOL\".\"PATH\" AS POOL_PATH, + \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME +FROM \"WM_POOL_TO_TRIGGER\" + JOIN \"WM_POOL\" ON \"WM_POOL_TO_TRIGGER\".\"POOL_ID\" = \"WM_POOL\".\"POOL_ID\" + JOIN \"WM_TRIGGER\" ON \"WM_POOL_TO_TRIGGER\".\"TRIGGER_ID\" = \"WM_TRIGGER\".\"TRIGGER_ID\" + JOIN \"WM_RESOURCEPLAN\" ON \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +UNION +SELECT + \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, + '' AS POOL_PATH, + \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME +FROM \"WM_TRIGGER\" + JOIN \"WM_RESOURCEPLAN\" ON \"WM_TRIGGER\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't') +" +); + +DROP TABLE IF EXISTS `WM_MAPPINGS`; +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_MAPPINGS` ( + `RP_NAME` string, + `NS` string, + `ENTITY_TYPE` string, + `ENTITY_NAME` string, + `POOL_PATH` string, + `ORDERING` int +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, + \"ENTITY_TYPE\", + \"ENTITY_NAME\", + case when \"WM_POOL\".\"PATH\" is null then '' else \"WM_POOL\".\"PATH\" end, + \"ORDERING\" +FROM \"WM_MAPPING\" +JOIN \"WM_RESOURCEPLAN\" ON \"WM_MAPPING\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_ID\" +" +); + + + DROP TABLE IF EXISTS `VERSION`; CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '4.0.0' AS `SCHEMA_VERSION`, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 807f159daa98d40e667914adc6c53fb8ecabf998..8eb9ebe607bad20f3b6c83e466242dc2959df83c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -695,6 +695,11 @@ private int showResourcePlans(Hive db, ShowResourcePlanDesc showResourcePlanDesc return 0; } + // Note: the resource plan operations are going to be annotated with namespace based on the config + // inside Hive.java. We don't want HS2 to be aware of namespaces beyond that, or to even see + // that there exist other namespaces, because one HS2 always operates inside just one and we + // don't want this complexity to bleed everywhere. Therefore, this code doesn't care about + // namespaces - Hive.java will transparently scope everything. That's the idea anyway. private int alterResourcePlan(Hive db, AlterResourcePlanDesc desc) throws HiveException { if (desc.shouldValidate()) { WMValidateResourcePlanResponse result = db.validateResourcePlan(desc.getResourcePlanName()); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 012a670064c155a31dab5337dc97ac5bb7b39aec..11d8f0cc26c5869acf6bb2828bf60e9a7f69caa1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -5637,9 +5637,15 @@ public void addCheckConstraint(List checkConstraints) } } - public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName, boolean ifNotExists) throws HiveException { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (resourcePlan.isSetNs() && !ns.equals(resourcePlan.getNs())) { + throw new HiveException("Cannot create a plan in a different NS; was " + + resourcePlan.getNs() + ", configured " + ns); + } + resourcePlan.setNs(ns); + try { getMSC().createResourcePlan(resourcePlan, copyFromName); } catch (AlreadyExistsException e) { @@ -5653,7 +5659,7 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName, public WMFullResourcePlan getResourcePlan(String rpName) throws HiveException { try { - return getMSC().getResourcePlan(rpName); + return getMSC().getResourcePlan(rpName, conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (NoSuchObjectException e) { return null; } catch (Exception e) { @@ -5663,7 +5669,7 @@ public WMFullResourcePlan getResourcePlan(String rpName) throws HiveException { public List getAllResourcePlans() throws HiveException { try { - return getMSC().getAllResourcePlans(); + return getMSC().getAllResourcePlans(conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (Exception e) { throw new HiveException(e); } @@ -5671,7 +5677,8 @@ public WMFullResourcePlan getResourcePlan(String rpName) throws HiveException { public void dropResourcePlan(String rpName, boolean ifExists) throws HiveException { try { - getMSC().dropResourcePlan(rpName); + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + getMSC().dropResourcePlan(rpName, ns); } catch (NoSuchObjectException e) { if (!ifExists) { throw new HiveException(e, ErrorMsg.RESOURCE_PLAN_NOT_EXISTS, rpName); @@ -5684,7 +5691,13 @@ public void dropResourcePlan(String rpName, boolean ifExists) throws HiveExcepti public WMFullResourcePlan alterResourcePlan(String rpName, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) throws HiveException { try { - return getMSC().alterResourcePlan(rpName, resourcePlan, canActivateDisabled, + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (resourcePlan.isSetNs() && !ns.equals(resourcePlan.getNs())) { + throw new HiveException("Cannot modify a plan in a different NS; was " + + resourcePlan.getNs() + ", configured " + ns); + } + resourcePlan.setNs(ns); + return getMSC().alterResourcePlan(rpName, ns, resourcePlan, canActivateDisabled, isForceDeactivate, isReplace); } catch (Exception e) { throw new HiveException(e); @@ -5693,7 +5706,8 @@ public WMFullResourcePlan alterResourcePlan(String rpName, WMNullableResourcePla public WMFullResourcePlan getActiveResourcePlan() throws HiveException { try { - return getMSC().getActiveResourcePlan(); + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + return getMSC().getActiveResourcePlan(ns); } catch (Exception e) { throw new HiveException(e); } @@ -5701,7 +5715,8 @@ public WMFullResourcePlan getActiveResourcePlan() throws HiveException { public WMValidateResourcePlanResponse validateResourcePlan(String rpName) throws HiveException { try { - return getMSC().validateResourcePlan(rpName); + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + return getMSC().validateResourcePlan(rpName, ns); } catch (Exception e) { throw new HiveException(e); } @@ -5709,6 +5724,12 @@ public WMValidateResourcePlanResponse validateResourcePlan(String rpName) throws public void createWMTrigger(WMTrigger trigger) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (trigger.isSetNs() && !ns.equals(trigger.getNs())) { + throw new HiveException("Cannot create a trigger in a different NS; was " + + trigger.getNs() + ", configured " + ns); + } + trigger.setNs(ns); getMSC().createWMTrigger(trigger); } catch (Exception e) { throw new HiveException(e); @@ -5717,6 +5738,12 @@ public void createWMTrigger(WMTrigger trigger) throws HiveException { public void alterWMTrigger(WMTrigger trigger) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (trigger.isSetNs() && !ns.equals(trigger.getNs())) { + throw new HiveException("Cannot modify a trigger in a different NS; was " + + trigger.getNs() + ", configured " + ns); + } + trigger.setNs(ns); getMSC().alterWMTrigger(trigger); } catch (Exception e) { throw new HiveException(e); @@ -5725,7 +5752,7 @@ public void alterWMTrigger(WMTrigger trigger) throws HiveException { public void dropWMTrigger(String rpName, String triggerName) throws HiveException { try { - getMSC().dropWMTrigger(rpName, triggerName); + getMSC().dropWMTrigger(rpName, triggerName, conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (Exception e) { throw new HiveException(e); } @@ -5733,6 +5760,12 @@ public void dropWMTrigger(String rpName, String triggerName) throws HiveExceptio public void createWMPool(WMPool pool) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (pool.isSetNs() && !ns.equals(pool.getNs())) { + throw new HiveException("Cannot create a pool in a different NS; was " + + pool.getNs() + ", configured " + ns); + } + pool.setNs(ns); getMSC().createWMPool(pool); } catch (Exception e) { throw new HiveException(e); @@ -5741,6 +5774,12 @@ public void createWMPool(WMPool pool) throws HiveException { public void alterWMPool(WMNullablePool pool, String poolPath) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (pool.isSetNs() && !ns.equals(pool.getNs())) { + throw new HiveException("Cannot modify a pool in a different NS; was " + + pool.getNs() + ", configured " + ns); + } + pool.setNs(ns); getMSC().alterWMPool(pool, poolPath); } catch (Exception e) { throw new HiveException(e); @@ -5749,7 +5788,8 @@ public void alterWMPool(WMNullablePool pool, String poolPath) throws HiveExcepti public void dropWMPool(String resourcePlanName, String poolPath) throws HiveException { try { - getMSC().dropWMPool(resourcePlanName, poolPath); + getMSC().dropWMPool(resourcePlanName, poolPath, + conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (Exception e) { throw new HiveException(e); } @@ -5758,6 +5798,12 @@ public void dropWMPool(String resourcePlanName, String poolPath) throws HiveExce public void createOrUpdateWMMapping(WMMapping mapping, boolean isUpdate) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (mapping.isSetNs() && !ns.equals(mapping.getNs())) { + throw new HiveException("Cannot create a mapping in a different NS; was " + + mapping.getNs() + ", configured " + ns); + } + mapping.setNs(ns); getMSC().createOrUpdateWMMapping(mapping, isUpdate); } catch (Exception e) { throw new HiveException(e); @@ -5766,17 +5812,24 @@ public void createOrUpdateWMMapping(WMMapping mapping, boolean isUpdate) public void dropWMMapping(WMMapping mapping) throws HiveException { try { + String ns = conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE); + if (mapping.isSetNs() && !ns.equals(mapping.getNs())) { + throw new HiveException("Cannot modify a mapping in a different NS; was " + + mapping.getNs() + ", configured " + ns); + } + mapping.setNs(ns); getMSC().dropWMMapping(mapping); } catch (Exception e) { throw new HiveException(e); } } - + // TODO: eh public void createOrDropTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, boolean shouldDrop) throws HiveException { try { - getMSC().createOrDropTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, shouldDrop); + getMSC().createOrDropTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, + shouldDrop, conf.getVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE)); } catch (Exception e) { throw new HiveException(e); } diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index e57db935d9420508ed6091e12ca6b6cd3382db5d..8d55fecbf98ef89a442c50fae851334a17f72729 100755 --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -26,6 +26,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; import org.apache.hadoop.fs.FileStatus; @@ -37,7 +38,13 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMPool; +import org.apache.hadoop.hive.metastore.api.WMResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMResourcePlanStatus; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.session.SessionState; @@ -59,8 +66,10 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import org.apache.thrift.protocol.TBinaryProtocol; import org.junit.Assert; +import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import junit.framework.TestCase; @@ -76,20 +85,21 @@ protected void setUp() throws Exception { super.setUp(); hiveConf = new HiveConf(this.getClass()); - hiveConf - .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + hm = setUpImpl(hiveConf); + } + + private static Hive setUpImpl(HiveConf hiveConf) throws Exception { + hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); // enable trash so it can be tested hiveConf.setFloat("fs.trash.checkpoint.interval", 30); // FS_TRASH_CHECKPOINT_INTERVAL_KEY (hadoop-2) hiveConf.setFloat("fs.trash.interval", 30); // FS_TRASH_INTERVAL_KEY (hadoop-2) SessionState.start(hiveConf); try { - hm = Hive.get(hiveConf); + return Hive.get(hiveConf); } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); - System.err - .println("Unable to initialize Hive Metastore using configuration: \n " - + hiveConf); + System.err.println("Unable to initialize Hive Metastore using configuration: \n" + hiveConf); throw e; } } @@ -422,6 +432,52 @@ public void testGetAndDropTables() throws Throwable { } } + public void testWmNamespaceHandling() throws Throwable { + HiveConf hiveConf = new HiveConf(this.getClass()); + Hive hm = setUpImpl(hiveConf); + // TODO: threadlocals... Why is all this Hive client stuff like that?!! + final AtomicReference hm2r = new AtomicReference<>(); + Thread pointlessThread = new Thread(new Runnable() { + @Override + public void run() { + HiveConf hiveConf2 = new HiveConf(this.getClass()); + hiveConf2.setVar(ConfVars.HIVE_SERVER2_WM_NAMESPACE, "hm2"); + try { + hm2r.set(setUpImpl(hiveConf2)); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + } + } + }); + pointlessThread.start(); + pointlessThread.join(); + Hive hm2 = hm2r.get(); + assertNotNull(hm2); + + hm.createResourcePlan(new WMResourcePlan("hm"), null, false); + assertEquals(1, hm.getAllResourcePlans().size()); + assertEquals(0, hm2.getAllResourcePlans().size()); + hm2.createResourcePlan(new WMResourcePlan("hm"), null, false); + WMNullableResourcePlan changes = new WMNullableResourcePlan(); + changes.setStatus(WMResourcePlanStatus.ACTIVE); + hm.alterResourcePlan("hm", changes, true, false, false); + // We should not be able to modify the active plan. + WMPool pool = new WMPool("hm", "foo"); + pool.setAllocFraction(0); + pool.setQueryParallelism(1); + try { + hm.createWMPool(pool); + fail("Expected exception"); + } catch (HiveException e) { + } + // But we should still be able to modify the other plan. + pool.unsetNs(); // The call to create sets the namespace. + hm2.createWMPool(pool); + // Make the 2nd plan active in a different namespace. + changes.unsetNs(); + hm2.alterResourcePlan("hm", changes, true, false, false); + } + public void testDropTableTrash() throws Throwable { if (!ShimLoader.getHadoopShims().supportTrashFeature()) { return; // it's hadoop-1 diff --git ql/src/test/queries/clientpositive/resourceplan.q ql/src/test/queries/clientpositive/resourceplan.q index fae9701ebaeaa521904a383f5fb741c13be08d8e..46aae72a1100f9efd568a606f08242013c6fc016 100644 --- ql/src/test/queries/clientpositive/resourceplan.q +++ ql/src/test/queries/clientpositive/resourceplan.q @@ -10,7 +10,7 @@ set hive.cbo.enable=false; show grant user hive_test_user; -- Initialize the hive schema. -source ../../metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql; +source ../../metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql; -- SORT_QUERY_RESULTS @@ -240,13 +240,13 @@ CREATE POOL plan_2.default.c1 WITH ALLOC_FRACTION=0.3, QUERY_PARALLELISM=3, SCHEDULING_POLICY='fair'; CREATE POOL plan_2.default.c2 WITH - QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.7; + QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.75; -- Cannot activate c1 + c2 = 1.0 ALTER RESOURCE PLAN plan_2 VALIDATE; ALTER RESOURCE PLAN plan_2 ENABLE ACTIVATE; -ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.5, QUERY_PARALLELISM = 1; +ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.7, QUERY_PARALLELISM = 1; ALTER POOL plan_2.default.c2 SET SCHEDULING_POLICY='fair'; SELECT * FROM SYS.WM_POOLS; ALTER POOL plan_2.default.c2 UNSET SCHEDULING_POLICY; diff --git ql/src/test/results/clientpositive/llap/resourceplan.q.out ql/src/test/results/clientpositive/llap/resourceplan.q.out index c11daf728cdd5bd6fe36618aff113b3d60579129..7fd72781e559a6054773987e7c4313a58df199e7 100644 --- ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -1838,14 +1838,14 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@PART_COL_STATS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`, - 'Hive release version 3.1.0' AS `VERSION_COMMENT` +PREHOOK: query: CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '4.0.0' AS `SCHEMA_VERSION`, + 'Hive release version 4.0.0' AS `VERSION_COMMENT` PREHOOK: type: CREATEVIEW PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: SYS@VERSION PREHOOK: Output: database:sys -POSTHOOK: query: CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`, - 'Hive release version 3.1.0' AS `VERSION_COMMENT` +POSTHOOK: query: CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '4.0.0' AS `SCHEMA_VERSION`, + 'Hive release version 4.0.0' AS `VERSION_COMMENT` POSTHOOK: type: CREATEVIEW POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: SYS@VERSION @@ -2107,6 +2107,7 @@ POSTHOOK: Lineage: PARTITION_STATS_VIEW.total_size EXPRESSION [(partition_params POSTHOOK: Lineage: PARTITION_STATS_VIEW.transient_last_ddl_time EXPRESSION [(partition_params)partition_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (partition_params)partition_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ] PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( `NAME` string, + `NS` string, `STATUS` string, `QUERY_PARALLELISM` int, `DEFAULT_POOL_PATH` string @@ -2117,6 +2118,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"STATUS\", \"WM_RESOURCEPLAN\".\"QUERY_PARALLELISM\", \"WM_POOL\".\"PATH\" @@ -2128,6 +2130,7 @@ PREHOOK: Output: SYS@WM_RESOURCEPLANS PREHOOK: Output: database:sys POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( `NAME` string, + `NS` string, `STATUS` string, `QUERY_PARALLELISM` int, `DEFAULT_POOL_PATH` string @@ -2138,6 +2141,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"STATUS\", \"WM_RESOURCEPLAN\".\"QUERY_PARALLELISM\", \"WM_POOL\".\"PATH\" @@ -2149,6 +2153,7 @@ POSTHOOK: Output: SYS@WM_RESOURCEPLANS POSTHOOK: Output: database:sys PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_TRIGGERS` ( `RP_NAME` string, + `NS` string, `NAME` string, `TRIGGER_EXPRESSION` string, `ACTION_EXPRESSION` string @@ -2159,6 +2164,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT r.\"NAME\" AS RP_NAME, + case when r.\"NS\" is null then 'default' else r.\"NS\" end, t.\"NAME\" AS NAME, \"TRIGGER_EXPRESSION\", \"ACTION_EXPRESSION\" @@ -2174,6 +2180,7 @@ PREHOOK: Output: SYS@WM_TRIGGERS PREHOOK: Output: database:sys POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_TRIGGERS` ( `RP_NAME` string, + `NS` string, `NAME` string, `TRIGGER_EXPRESSION` string, `ACTION_EXPRESSION` string @@ -2184,6 +2191,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT r.\"NAME\" AS RP_NAME, + case when r.\"NS\" is null then 'default' else r.\"NS\" end, t.\"NAME\" AS NAME, \"TRIGGER_EXPRESSION\", \"ACTION_EXPRESSION\" @@ -2199,6 +2207,7 @@ POSTHOOK: Output: SYS@WM_TRIGGERS POSTHOOK: Output: database:sys PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS` ( `RP_NAME` string, + `NS` string, `PATH` string, `ALLOC_FRACTION` double, `QUERY_PARALLELISM` int, @@ -2210,6 +2219,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"WM_POOL\".\"PATH\", \"WM_POOL\".\"ALLOC_FRACTION\", \"WM_POOL\".\"QUERY_PARALLELISM\", @@ -2226,6 +2236,7 @@ PREHOOK: Output: SYS@WM_POOLS PREHOOK: Output: database:sys POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS` ( `RP_NAME` string, + `NS` string, `PATH` string, `ALLOC_FRACTION` double, `QUERY_PARALLELISM` int, @@ -2237,6 +2248,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"WM_POOL\".\"PATH\", \"WM_POOL\".\"ALLOC_FRACTION\", \"WM_POOL\".\"QUERY_PARALLELISM\", @@ -2253,6 +2265,7 @@ POSTHOOK: Output: SYS@WM_POOLS POSTHOOK: Output: database:sys PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( `RP_NAME` string, + `NS` string, `POOL_PATH` string, `TRIGGER_NAME` string ) @@ -2262,6 +2275,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"WM_POOL\".\"PATH\" AS POOL_PATH, \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME FROM \"WM_POOL_TO_TRIGGER\" @@ -2271,6 +2285,7 @@ FROM \"WM_POOL_TO_TRIGGER\" UNION SELECT \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, '' AS POOL_PATH, \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME FROM \"WM_TRIGGER\" @@ -2283,6 +2298,7 @@ PREHOOK: Output: SYS@WM_POOLS_TO_TRIGGERS PREHOOK: Output: database:sys POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( `RP_NAME` string, + `NS` string, `POOL_PATH` string, `TRIGGER_NAME` string ) @@ -2292,6 +2308,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"WM_POOL\".\"PATH\" AS POOL_PATH, \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME FROM \"WM_POOL_TO_TRIGGER\" @@ -2301,6 +2318,7 @@ FROM \"WM_POOL_TO_TRIGGER\" UNION SELECT \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, '' AS POOL_PATH, \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME FROM \"WM_TRIGGER\" @@ -2313,6 +2331,7 @@ POSTHOOK: Output: SYS@WM_POOLS_TO_TRIGGERS POSTHOOK: Output: database:sys PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_MAPPINGS` ( `RP_NAME` string, + `NS` string, `ENTITY_TYPE` string, `ENTITY_NAME` string, `POOL_PATH` string, @@ -2324,6 +2343,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"ENTITY_TYPE\", \"ENTITY_NAME\", case when \"WM_POOL\".\"PATH\" is null then '' else \"WM_POOL\".\"PATH\" end, @@ -2338,6 +2358,7 @@ PREHOOK: Output: SYS@WM_MAPPINGS PREHOOK: Output: database:sys POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_MAPPINGS` ( `RP_NAME` string, + `NS` string, `ENTITY_TYPE` string, `ENTITY_NAME` string, `POOL_PATH` string, @@ -2349,6 +2370,7 @@ TBLPROPERTIES ( "hive.sql.query" = "SELECT \"WM_RESOURCEPLAN\".\"NAME\", + case when \"WM_RESOURCEPLAN\".\"NS\" is null then 'default' else \"WM_RESOURCEPLAN\".\"NS\" end AS NS, \"ENTITY_TYPE\", \"ENTITY_NAME\", case when \"WM_POOL\".\"PATH\" is null then '' else \"WM_POOL\".\"PATH\" end, @@ -2392,10 +2414,11 @@ SELECT DISTINCT cast(null as string), `DB_LOCATION_URI` FROM - `sys`.`DBS` D LEFT JOIN `sys`.`TBLS` T ON (D.`DB_ID` = T.`DB_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P ON (T.`TBL_ID` = P.`TBL_ID`) + `sys`.`DBS` D, `sys`.`TBLS` T, `sys`.`TBL_PRIVS` P WHERE - NOT restrict_information_schema() OR P.`TBL_ID` IS NOT NULL + NOT restrict_information_schema() OR + D.`DB_ID` = T.`DB_ID` + AND T.`TBL_ID` = P.`TBL_ID` AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) AND current_authorizer() = P.`AUTHORIZER` @@ -2424,10 +2447,11 @@ SELECT DISTINCT cast(null as string), `DB_LOCATION_URI` FROM - `sys`.`DBS` D LEFT JOIN `sys`.`TBLS` T ON (D.`DB_ID` = T.`DB_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P ON (T.`TBL_ID` = P.`TBL_ID`) + `sys`.`DBS` D, `sys`.`TBLS` T, `sys`.`TBL_PRIVS` P WHERE - NOT restrict_information_schema() OR P.`TBL_ID` IS NOT NULL + NOT restrict_information_schema() OR + D.`DB_ID` = T.`DB_ID` + AND T.`TBL_ID` = P.`TBL_ID` AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) AND current_authorizer() = P.`AUTHORIZER` @@ -2473,12 +2497,12 @@ SELECT DISTINCT 'NO', cast(null as string) FROM - `sys`.`TBLS` T JOIN `sys`.`DBS` D ON (D.`DB_ID` = T.`DB_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P ON (T.`TBL_ID` = P.`TBL_ID`) + `sys`.`TBLS` T, `sys`.`DBS` D, `sys`.`TBL_PRIVS` P WHERE - NOT restrict_information_schema() OR P.`TBL_ID` IS NOT NULL + D.`DB_ID` = T.`DB_ID` + AND (NOT restrict_information_schema() OR T.`TBL_ID` = P.`TBL_ID` AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' - OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))) AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@dbs @@ -2515,12 +2539,12 @@ SELECT DISTINCT 'NO', cast(null as string) FROM - `sys`.`TBLS` T JOIN `sys`.`DBS` D ON (D.`DB_ID` = T.`DB_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P ON (T.`TBL_ID` = P.`TBL_ID`) + `sys`.`TBLS` T, `sys`.`DBS` D, `sys`.`TBL_PRIVS` P WHERE - NOT restrict_information_schema() OR P.`TBL_ID` IS NOT NULL + D.`DB_ID` = T.`DB_ID` + AND (NOT restrict_information_schema() OR T.`TBL_ID` = P.`TBL_ID` AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' - OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))) AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() POSTHOOK: type: CREATEVIEW POSTHOOK: Input: sys@dbs @@ -2561,15 +2585,18 @@ SELECT DISTINCT IF (P.`GRANT_OPTION` == 0, 'NO', 'YES'), 'NO' FROM - `sys`.`TBL_PRIVS` P JOIN `sys`.`TBLS` T ON (P.`TBL_ID` = T.`TBL_ID`) - JOIN `sys`.`DBS` D ON (T.`DB_ID` = D.`DB_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P2 ON (P.`TBL_ID` = P2.`TBL_ID`) + `sys`.`TBL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_PRIVS` P2 WHERE - NOT restrict_information_schema() OR - (P2.`TBL_ID` IS NOT NULL AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + P.`TBL_ID` = T.`TBL_ID` + AND T.`DB_ID` = D.`DB_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' - OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) - AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER` = current_authorizer() AND P2.`AUTHORIZER` = current_authorizer()) + OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP'))) + AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER` = current_authorizer() AND P2.`AUTHORIZER` = current_authorizer() PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@dbs PREHOOK: Input: sys@tbl_privs @@ -2597,15 +2624,18 @@ SELECT DISTINCT IF (P.`GRANT_OPTION` == 0, 'NO', 'YES'), 'NO' FROM - `sys`.`TBL_PRIVS` P JOIN `sys`.`TBLS` T ON (P.`TBL_ID` = T.`TBL_ID`) - JOIN `sys`.`DBS` D ON (T.`DB_ID` = D.`DB_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P2 ON (P.`TBL_ID` = P2.`TBL_ID`) + `sys`.`TBL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_PRIVS` P2 WHERE - NOT restrict_information_schema() OR - (P2.`TBL_ID` IS NOT NULL AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + P.`TBL_ID` = T.`TBL_ID` + AND T.`DB_ID` = D.`DB_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' - OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) - AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER` = current_authorizer() AND P2.`AUTHORIZER` = current_authorizer()) + OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP'))) + AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER` = current_authorizer() AND P2.`AUTHORIZER` = current_authorizer() POSTHOOK: type: CREATEVIEW POSTHOOK: Input: sys@dbs POSTHOOK: Input: sys@tbl_privs @@ -2759,16 +2789,21 @@ SELECT DISTINCT WHEN lower(C.TYPE_NAME) like 'numeric%' THEN 10 ELSE null END FROM - `sys`.`COLUMNS_V2` C JOIN `sys`.`SDS` S ON (C.`CD_ID` = S.`CD_ID`) - JOIN `sys`.`TBLS` T ON (S.`SD_ID` = T.`SD_ID`) - JOIN `sys`.`DBS` D ON (T.`DB_ID` = D.`DB_ID`) - LEFT JOIN `sys`.`TBL_COL_PRIVS` P ON (T.`TBL_ID` = P.`TBL_ID`) + `sys`.`COLUMNS_V2` C, + `sys`.`SDS` S, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_COL_PRIVS` P WHERE - NOT restrict_information_schema() OR P.`TBL_ID` IS NOT NULL + S.`SD_ID` = T.`SD_ID` + AND T.`DB_ID` = D.`DB_ID` + AND C.`CD_ID` = S.`CD_ID` + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` AND C.`COLUMN_NAME` = P.`COLUMN_NAME` AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) - AND P.`TBL_COL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() + AND P.`TBL_COL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer()) PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@columns_v2 PREHOOK: Input: sys@dbs @@ -2916,16 +2951,21 @@ SELECT DISTINCT WHEN lower(C.TYPE_NAME) like 'numeric%' THEN 10 ELSE null END FROM - `sys`.`COLUMNS_V2` C JOIN `sys`.`SDS` S ON (C.`CD_ID` = S.`CD_ID`) - JOIN `sys`.`TBLS` T ON (S.`SD_ID` = T.`SD_ID`) - JOIN `sys`.`DBS` D ON (T.`DB_ID` = D.`DB_ID`) - LEFT JOIN `sys`.`TBL_COL_PRIVS` P ON (T.`TBL_ID` = P.`TBL_ID`) + `sys`.`COLUMNS_V2` C, + `sys`.`SDS` S, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_COL_PRIVS` P WHERE - NOT restrict_information_schema() OR P.`TBL_ID` IS NOT NULL + S.`SD_ID` = T.`SD_ID` + AND T.`DB_ID` = D.`DB_ID` + AND C.`CD_ID` = S.`CD_ID` + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` AND C.`COLUMN_NAME` = P.`COLUMN_NAME` AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) - AND P.`TBL_COL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() + AND P.`TBL_COL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer()) POSTHOOK: type: CREATEVIEW POSTHOOK: Input: sys@columns_v2 POSTHOOK: Input: sys@dbs @@ -3002,16 +3042,20 @@ SELECT DISTINCT P.`TBL_COL_PRIV`, IF (P.`GRANT_OPTION` == 0, 'NO', 'YES') FROM - `sys`.`TBL_COL_PRIVS` P JOIN `sys`.`TBLS` T ON (P.`TBL_ID` = T.`TBL_ID`) - JOIN `sys`.`DBS` D ON (T.`DB_ID` = D.`DB_ID`) - JOIN `sys`.`SDS` S ON (S.`SD_ID` = T.`SD_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P2 ON (P.`TBL_ID` = P2.`TBL_ID`) + `sys`.`TBL_COL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`SDS` S, + `sys`.`TBL_PRIVS` P2 WHERE - NOT restrict_information_schema() OR P2.`TBL_ID` IS NOT NULL - AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + S.`SD_ID` = T.`SD_ID` + AND T.`DB_ID` = D.`DB_ID` + AND P.`TBL_ID` = T.`TBL_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) - AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() AND P2.`AUTHORIZER`=current_authorizer() + AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() AND P2.`AUTHORIZER`=current_authorizer()) PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@dbs PREHOOK: Input: sys@sds @@ -3041,16 +3085,20 @@ SELECT DISTINCT P.`TBL_COL_PRIV`, IF (P.`GRANT_OPTION` == 0, 'NO', 'YES') FROM - `sys`.`TBL_COL_PRIVS` P JOIN `sys`.`TBLS` T ON (P.`TBL_ID` = T.`TBL_ID`) - JOIN `sys`.`DBS` D ON (T.`DB_ID` = D.`DB_ID`) - JOIN `sys`.`SDS` S ON (S.`SD_ID` = T.`SD_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P2 ON (P.`TBL_ID` = P2.`TBL_ID`) + `sys`.`TBL_COL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`SDS` S, + `sys`.`TBL_PRIVS` P2 WHERE - NOT restrict_information_schema() OR P2.`TBL_ID` IS NOT NULL - AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + S.`SD_ID` = T.`SD_ID` + AND T.`DB_ID` = D.`DB_ID` + AND P.`TBL_ID` = T.`TBL_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) - AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() AND P2.`AUTHORIZER`=current_authorizer() + AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() AND P2.`AUTHORIZER`=current_authorizer()) POSTHOOK: type: CREATEVIEW POSTHOOK: Input: sys@dbs POSTHOOK: Input: sys@sds @@ -3092,11 +3140,14 @@ SELECT DISTINCT false, false FROM - `sys`.`DBS` D JOIN `sys`.`TBLS` T ON (D.`DB_ID` = T.`DB_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P ON (T.`TBL_ID` = P.`TBL_ID`) + `sys`.`DBS` D, + `sys`.`TBLS` T, + `sys`.`TBL_PRIVS` P WHERE - length(T.VIEW_ORIGINAL_TEXT) > 0 - AND (NOT restrict_information_schema() OR P.`TBL_ID` IS NOT NULL + D.`DB_ID` = T.`DB_ID` + AND length(T.VIEW_ORIGINAL_TEXT) > 0 + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer()) @@ -3131,11 +3182,14 @@ SELECT DISTINCT false, false FROM - `sys`.`DBS` D JOIN `sys`.`TBLS` T ON (D.`DB_ID` = T.`DB_ID`) - LEFT JOIN `sys`.`TBL_PRIVS` P ON (T.`TBL_ID` = P.`TBL_ID`) + `sys`.`DBS` D, + `sys`.`TBLS` T, + `sys`.`TBL_PRIVS` P WHERE - length(T.VIEW_ORIGINAL_TEXT) > 0 - AND (NOT restrict_information_schema() OR P.`TBL_ID` IS NOT NULL + D.`DB_ID` = T.`DB_ID` + AND length(T.VIEW_ORIGINAL_TEXT) > 0 + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer()) @@ -3195,7 +3249,7 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 DISABLED NULL default +plan_1 default DISABLED NULL default PREHOOK: query: CREATE RESOURCE PLAN plan_2 WITH QUERY_PARALLELISM=5 PREHOOK: type: CREATE RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3229,8 +3283,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 DISABLED NULL default -plan_2 DISABLED 10 default +plan_1 default DISABLED NULL default +plan_2 default DISABLED 10 default PREHOOK: query: CREATE RESOURCE PLAN plan_2 PREHOOK: type: CREATE RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3253,8 +3307,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 DISABLED NULL default -plan_2 DISABLED 10 default +plan_1 default DISABLED NULL default +plan_2 default DISABLED 10 default PREHOOK: query: ALTER RESOURCE PLAN plan_1 RENAME TO plan_3 PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3268,8 +3322,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED NULL default +plan_2 default DISABLED 10 default +plan_3 default DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 4 PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3283,8 +3337,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED 4 default +plan_2 default DISABLED 10 default +plan_3 default DISABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 UNSET QUERY_PARALLELISM PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3298,8 +3352,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED NULL default +plan_2 default DISABLED 10 default +plan_3 default DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 30, DEFAULT POOL = default1 PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3312,8 +3366,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED NULL default +plan_2 default DISABLED 10 default +plan_3 default DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3340,8 +3394,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED NULL default +plan_2 default DISABLED 10 default +plan_3 default DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3354,8 +3408,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED NULL default +plan_2 default DISABLED 10 default +plan_3 default DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3369,8 +3423,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED NULL default +plan_2 default DISABLED 10 default +plan_3 default DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3384,8 +3438,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ENABLED NULL default +plan_2 default DISABLED 10 default +plan_3 default ENABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3399,8 +3453,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ACTIVE NULL default +plan_2 default DISABLED 10 default +plan_3 default ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3414,8 +3468,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ACTIVE NULL default +plan_2 default DISABLED 10 default +plan_3 default ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3428,8 +3482,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ACTIVE NULL default +plan_2 default DISABLED 10 default +plan_3 default ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3442,8 +3496,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ACTIVE NULL default +plan_2 default DISABLED 10 default +plan_3 default ACTIVE NULL default PREHOOK: query: DISABLE WORKLOAD MANAGEMENT PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3457,8 +3511,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ENABLED NULL default +plan_2 default DISABLED 10 default +plan_3 default ENABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3477,8 +3531,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ACTIVE NULL default +plan_2 default DISABLED 10 default +plan_3 default ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_2 ENABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3492,8 +3546,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ENABLED 10 default -plan_3 ACTIVE NULL default +plan_2 default ENABLED 10 default +plan_3 default ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_2 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3507,8 +3561,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default -plan_3 ENABLED NULL default +plan_2 default ACTIVE 10 default +plan_3 default ENABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3522,8 +3576,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default -plan_3 ENABLED NULL default +plan_2 default ACTIVE 10 default +plan_3 default ENABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3537,8 +3591,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default -plan_3 DISABLED NULL default +plan_2 default ACTIVE 10 default +plan_3 default DISABLED NULL default PREHOOK: query: DROP RESOURCE PLAN plan_2 PREHOOK: type: DROP RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3556,7 +3610,7 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default +plan_2 default ACTIVE 10 default PREHOOK: query: DROP RESOURCE PLAN plan_99999 PREHOOK: type: DROP RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3584,8 +3638,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default -table DISABLED 1 default +plan_2 default ACTIVE 10 default +table default DISABLED 1 default PREHOOK: query: create table wm_test(key string) PREHOOK: type: CREATETABLE PREHOOK: Output: INFORMATION_SCHEMA@wm_test @@ -3636,7 +3690,7 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_triggers #### A masked pattern was here #### -plan_1 trigger_1 BYTES_READ > '10kb' KILL +plan_1 default trigger_1 BYTES_READ > '10kb' KILL PREHOOK: query: CREATE TRIGGER plan_1.trigger_1 WHEN ELAPSED_TIME > 300 DO KILL PREHOOK: type: CREATE TRIGGER PREHOOK: Output: dummyHostnameForTest @@ -3660,8 +3714,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_triggers #### A masked pattern was here #### -plan_1 trigger_1 BYTES_READ > '10kb' KILL -plan_1 trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool +plan_1 default trigger_1 BYTES_READ > '10kb' KILL +plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool PREHOOK: query: ALTER TRIGGER plan_1.trigger_1 WHEN BYTES_READ > '1min' DO KILL PREHOOK: type: ALTER TRIGGER PREHOOK: Output: dummyHostnameForTest @@ -3675,8 +3729,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_triggers #### A masked pattern was here #### -plan_1 trigger_1 BYTES_READ > '1min' KILL -plan_1 trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool +plan_1 default trigger_1 BYTES_READ > '1min' KILL +plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool PREHOOK: query: DROP TRIGGER plan_1.trigger_1 PREHOOK: type: DROP TRIGGER PREHOOK: Output: dummyHostnameForTest @@ -3690,7 +3744,7 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_triggers #### A masked pattern was here #### -plan_1 trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool +plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool PREHOOK: query: CREATE TRIGGER plan_2.trigger_1 WHEN BYTES_READ > '100mb' DO MOVE TO null_pool PREHOOK: type: CREATE TRIGGER PREHOOK: Output: dummyHostnameForTest @@ -3728,12 +3782,12 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_triggers #### A masked pattern was here #### -plan_1 trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool -table database BYTES_WRITTEN > "1GB" MOVE TO default -table table BYTES_WRITTEN > '100KB' MOVE TO table -table trigger BYTES_WRITTEN > '100MB' MOVE TO default -table trigger1 ELAPSED_TIME > 10 KILL -table trigger2 ELAPSED_TIME > '1hour' KILL +plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool +table default database BYTES_WRITTEN > "1GB" MOVE TO default +table default table BYTES_WRITTEN > '100KB' MOVE TO table +table default trigger BYTES_WRITTEN > '100MB' MOVE TO default +table default trigger1 ELAPSED_TIME > 10 KILL +table default trigger2 ELAPSED_TIME > '1hour' KILL PREHOOK: query: DROP TRIGGER `table`.`database` PREHOOK: type: DROP TRIGGER PREHOOK: Output: dummyHostnameForTest @@ -3747,11 +3801,11 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_triggers #### A masked pattern was here #### -plan_1 trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool -table table BYTES_WRITTEN > '100KB' MOVE TO table -table trigger BYTES_WRITTEN > '100MB' MOVE TO default -table trigger1 ELAPSED_TIME > 10 KILL -table trigger2 ELAPSED_TIME > '1hour' KILL +plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool +table default table BYTES_WRITTEN > '100KB' MOVE TO table +table default trigger BYTES_WRITTEN > '100MB' MOVE TO default +table default trigger1 ELAPSED_TIME > 10 KILL +table default trigger2 ELAPSED_TIME > '1hour' KILL PREHOOK: query: ALTER RESOURCE PLAN plan_1 ENABLE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3765,9 +3819,9 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 ENABLED NULL default -plan_2 ACTIVE 10 default -table DISABLED 1 default +plan_1 default ENABLED NULL default +plan_2 default ACTIVE 10 default +table default DISABLED 1 default PREHOOK: query: DROP TRIGGER plan_1.trigger_2 PREHOOK: type: DROP TRIGGER PREHOOK: Output: dummyHostnameForTest @@ -3789,9 +3843,9 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 ACTIVE NULL default -plan_2 ENABLED 10 default -table DISABLED 1 default +plan_1 default ACTIVE NULL default +plan_2 default ENABLED 10 default +table default DISABLED 1 default PREHOOK: query: DROP TRIGGER plan_1.trigger_2 PREHOOK: type: DROP TRIGGER PREHOOK: Output: dummyHostnameForTest @@ -3818,12 +3872,12 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_triggers #### A masked pattern was here #### -plan_1 trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool -plan_2 trigger_1 BYTES_READ > 0 MOVE TO null_pool -table table BYTES_WRITTEN > '100KB' MOVE TO table -table trigger BYTES_WRITTEN > '100MB' MOVE TO default -table trigger1 ELAPSED_TIME > 10 KILL -table trigger2 ELAPSED_TIME > '1hour' KILL +plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool +plan_2 default trigger_1 BYTES_READ > 0 MOVE TO null_pool +table default table BYTES_WRITTEN > '100KB' MOVE TO table +table default trigger BYTES_WRITTEN > '100MB' MOVE TO default +table default trigger1 ELAPSED_TIME > 10 KILL +table default trigger2 ELAPSED_TIME > '1hour' KILL PREHOOK: query: CREATE POOL plan_1.default WITH ALLOC_FRACTION=1.0, QUERY_PARALLELISM=5, SCHEDULING_POLICY='default' PREHOOK: type: CREATE POOL @@ -3843,9 +3897,9 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 default 1.0 5 NULL -table default 1.0 4 NULL +plan_1 default default 1.0 4 NULL +plan_2 default default 1.0 5 NULL +table default default 1.0 4 NULL FAILED: SemanticException Invalid scheduling policy invalid PREHOOK: query: CREATE POOL plan_2.default.c1 WITH ALLOC_FRACTION=0.3, QUERY_PARALLELISM=3, SCHEDULING_POLICY='fair' @@ -3855,26 +3909,26 @@ POSTHOOK: query: CREATE POOL plan_2.default.c1 WITH ALLOC_FRACTION=0.3, QUERY_PARALLELISM=3, SCHEDULING_POLICY='fair' POSTHOOK: type: CREATE POOL PREHOOK: query: CREATE POOL plan_2.default.c2 WITH - QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.7 + QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.75 PREHOOK: type: CREATE POOL PREHOOK: Output: dummyHostnameForTest POSTHOOK: query: CREATE POOL plan_2.default.c2 WITH - QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.7 + QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.75 POSTHOOK: type: CREATE POOL PREHOOK: query: ALTER RESOURCE PLAN plan_2 VALIDATE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest POSTHOOK: query: ALTER RESOURCE PLAN plan_2 VALIDATE POSTHOOK: type: ALTER RESOURCEPLAN -Sum of children pools' alloc fraction should be less than 1 got: 1.0 for pool: default +Sum of children pools' alloc fraction should be less than 1 got: 1.05 for pool: default PREHOOK: query: ALTER RESOURCE PLAN plan_2 ENABLE ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:ResourcePlan: plan_2 is invalid: [Sum of children pools' alloc fraction should be less than 1 got: 1.0 for pool: default]) -PREHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.5, QUERY_PARALLELISM = 1 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:ResourcePlan: plan_2 is invalid: [Sum of children pools' alloc fraction should be less than 1 got: 1.05 for pool: default]) +PREHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.7, QUERY_PARALLELISM = 1 PREHOOK: type: ALTER POOL PREHOOK: Output: dummyHostnameForTest -POSTHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.5, QUERY_PARALLELISM = 1 +POSTHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.7, QUERY_PARALLELISM = 1 POSTHOOK: type: ALTER POOL PREHOOK: query: ALTER POOL plan_2.default.c2 SET SCHEDULING_POLICY='fair' PREHOOK: type: ALTER POOL @@ -3889,11 +3943,11 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 default 1.0 5 NULL -plan_2 default.c1 0.3 3 fair -plan_2 default.c2 0.5 1 fair -table default 1.0 4 NULL +plan_1 default default 1.0 4 NULL +plan_2 default default 1.0 5 NULL +plan_2 default default.c1 0.3 3 fair +plan_2 default default.c2 0.7 1 fair +table default default 1.0 4 NULL PREHOOK: query: ALTER POOL plan_2.default.c2 UNSET SCHEDULING_POLICY PREHOOK: type: ALTER POOL PREHOOK: Output: dummyHostnameForTest @@ -3907,11 +3961,11 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 default 1.0 5 NULL -plan_2 default.c1 0.3 3 fair -plan_2 default.c2 0.5 1 NULL -table default 1.0 4 NULL +plan_1 default default 1.0 4 NULL +plan_2 default default 1.0 5 NULL +plan_2 default default.c1 0.3 3 fair +plan_2 default default.c2 0.7 1 NULL +table default default 1.0 4 NULL PREHOOK: query: ALTER RESOURCE PLAN plan_2 VALIDATE PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -3946,11 +4000,11 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 def 1.0 5 NULL -plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.5 1 NULL -table default 1.0 4 NULL +plan_1 default default 1.0 4 NULL +plan_2 default def 1.0 5 NULL +plan_2 default def.c1 0.3 3 fair +plan_2 default def.c2 0.7 1 NULL +table default default 1.0 4 NULL PREHOOK: query: DROP POOL plan_2.default PREHOOK: type: DROP POOL PREHOOK: Output: dummyHostnameForTest @@ -3963,11 +4017,11 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 def 1.0 5 NULL -plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.5 1 NULL -table default 1.0 4 NULL +plan_1 default default 1.0 4 NULL +plan_2 default def 1.0 5 NULL +plan_2 default def.c1 0.3 3 fair +plan_2 default def.c2 0.7 1 NULL +table default default 1.0 4 NULL PREHOOK: query: CREATE POOL plan_2.child1.child2 WITH QUERY_PARALLELISM=2, SCHEDULING_POLICY='fifo', ALLOC_FRACTION=0.8 PREHOOK: type: CREATE POOL @@ -4014,15 +4068,15 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 def 1.0 5 NULL -plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.5 1 NULL -table default 1.0 4 NULL -table table 0.0 1 fifo -table table.pool1 0.9 3 fair -table table.pool1.child1 0.3 1 fair -table table.pool1.child2 0.7 3 fair +plan_1 default default 1.0 4 NULL +plan_2 default def 1.0 5 NULL +plan_2 default def.c1 0.3 3 fair +plan_2 default def.c2 0.7 1 NULL +table default default 1.0 4 NULL +table default table 0.0 1 fifo +table default table.pool1 0.9 3 fair +table default table.pool1.child1 0.3 1 fair +table default table.pool1.child2 0.7 3 fair PREHOOK: query: ALTER POOL `table`.`table`.pool1 SET PATH = `table`.pool PREHOOK: type: ALTER POOL PREHOOK: Output: dummyHostnameForTest @@ -4036,19 +4090,19 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 def 1.0 5 NULL -plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.5 1 NULL -table default 1.0 4 NULL -table table 0.0 1 fifo -table table.pool 0.9 3 fair -table table.pool.child1 0.3 1 fair -table table.pool.child2 0.7 3 fair +plan_1 default default 1.0 4 NULL +plan_2 default def 1.0 5 NULL +plan_2 default def.c1 0.3 3 fair +plan_2 default def.c2 0.7 1 NULL +table default default 1.0 4 NULL +table default table 0.0 1 fifo +table default table.pool 0.9 3 fair +table default table.pool.child1 0.3 1 fair +table default table.pool.child2 0.7 3 fair PREHOOK: query: DROP POOL `table`.`table` PREHOOK: type: DROP POOL PREHOOK: Output: dummyHostnameForTest -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:Pool has children cannot drop.) +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:Cannot drop a pool that has child pools) PREHOOK: query: SELECT * FROM SYS.WM_POOLS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools @@ -4057,15 +4111,15 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 def 1.0 5 NULL -plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.5 1 NULL -table default 1.0 4 NULL -table table 0.0 1 fifo -table table.pool 0.9 3 fair -table table.pool.child1 0.3 1 fair -table table.pool.child2 0.7 3 fair +plan_1 default default 1.0 4 NULL +plan_2 default def 1.0 5 NULL +plan_2 default def.c1 0.3 3 fair +plan_2 default def.c2 0.7 1 NULL +table default default 1.0 4 NULL +table default table 0.0 1 fifo +table default table.pool 0.9 3 fair +table default table.pool.child1 0.3 1 fair +table default table.pool.child2 0.7 3 fair PREHOOK: query: DROP POOL `table`.default PREHOOK: type: DROP POOL PREHOOK: Output: dummyHostnameForTest @@ -4078,15 +4132,15 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 def 1.0 5 NULL -plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.5 1 NULL -table default 1.0 4 NULL -table table 0.0 1 fifo -table table.pool 0.9 3 fair -table table.pool.child1 0.3 1 fair -table table.pool.child2 0.7 3 fair +plan_1 default default 1.0 4 NULL +plan_2 default def 1.0 5 NULL +plan_2 default def.c1 0.3 3 fair +plan_2 default def.c2 0.7 1 NULL +table default default 1.0 4 NULL +table default table 0.0 1 fifo +table default table.pool 0.9 3 fair +table default table.pool.child1 0.3 1 fair +table default table.pool.child2 0.7 3 fair PREHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_resourceplans @@ -4095,9 +4149,9 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 ACTIVE NULL default -plan_2 DISABLED 10 def -table DISABLED 1 default +plan_1 default ACTIVE NULL default +plan_2 default DISABLED 10 def +table default DISABLED 1 default PREHOOK: query: ALTER RESOURCE PLAN `table` SET DEFAULT POOL = `table`.pool PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -4116,14 +4170,14 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 def 1.0 5 NULL -plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.5 1 NULL -table table 0.0 1 fifo -table table.pool 0.9 3 fair -table table.pool.child1 0.3 1 fair -table table.pool.child2 0.7 3 fair +plan_1 default default 1.0 4 NULL +plan_2 default def 1.0 5 NULL +plan_2 default def.c1 0.3 3 fair +plan_2 default def.c2 0.7 1 NULL +table default table 0.0 1 fifo +table default table.pool 0.9 3 fair +table default table.pool.child1 0.3 1 fair +table default table.pool.child2 0.7 3 fair PREHOOK: query: ALTER RESOURCE PLAN `table` UNSET DEFAULT POOL PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -4137,9 +4191,9 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 ACTIVE NULL default -plan_2 DISABLED 10 def -table DISABLED 1 NULL +plan_1 default ACTIVE NULL default +plan_2 default DISABLED 10 def +table default DISABLED 1 NULL PREHOOK: query: ALTER POOL plan_2.def.c1 ADD TRIGGER trigger_1 PREHOOK: type: ALTER POOL PREHOOK: Output: dummyHostnameForTest @@ -4188,14 +4242,14 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS_TO_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools_to_triggers #### A masked pattern was here #### -plan_2 def.c1 trigger_1 -plan_2 def.c2 trigger_1 -table trigger1 -table table table -table table.pool.child1 table -table table.pool.child1 trigger1 -table table.pool.child2 trigger1 -table table.pool.child2 trigger2 +plan_2 default def.c1 trigger_1 +plan_2 default def.c2 trigger_1 +table default trigger1 +table default table table +table default table.pool.child1 table +table default table.pool.child1 trigger1 +table default table.pool.child2 trigger1 +table default table.pool.child2 trigger2 PREHOOK: query: SHOW RESOURCE PLAN `table` PREHOOK: type: SHOW RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -4233,12 +4287,12 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS_TO_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools_to_triggers #### A masked pattern was here #### -plan_2 def.c1 trigger_1 -plan_2 def.c2 trigger_1 -table table table -table table.pool.child1 table -table table.pool.child1 trigger1 -table table.pool.child2 trigger2 +plan_2 default def.c1 trigger_1 +plan_2 default def.c2 trigger_1 +table default table table +table default table.pool.child1 table +table default table.pool.child1 trigger1 +table default table.pool.child2 trigger2 PREHOOK: query: ALTER POOL plan_2.default ADD TRIGGER trigger_1 PREHOOK: type: ALTER POOL PREHOOK: Output: dummyHostnameForTest @@ -4255,12 +4309,12 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS_TO_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools_to_triggers #### A masked pattern was here #### -plan_2 def.c1 trigger_1 -plan_2 def.c2 trigger_1 -table table table -table table.pool.child1 table -table table.pool.child1 trigger1 -table table.pool.child2 trigger2 +plan_2 default def.c1 trigger_1 +plan_2 default def.c2 trigger_1 +table default table table +table default table.pool.child1 table +table default table.pool.child1 trigger1 +table default table.pool.child2 trigger2 PREHOOK: query: ALTER POOL plan_2.def.c1 DROP TRIGGER trigger_1 PREHOOK: type: ALTER POOL PREHOOK: Output: dummyHostnameForTest @@ -4288,8 +4342,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS_TO_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools_to_triggers #### A masked pattern was here #### -plan_2 def.c2 trigger_1 -table table table +plan_2 default def.c2 trigger_1 +table default table table PREHOOK: query: CREATE USER MAPPING "user1" IN plan_2 TO def PREHOOK: type: CREATE MAPPING PREHOOK: Output: dummyHostnameForTest @@ -4334,7 +4388,7 @@ plan_2[status=DISABLED,parallelism=10,defaultPool=def] + def[allocFraction=1.0,schedulingPolicy=null,parallelism=5] | mapped for users: user2 | mapped for default - + c2[allocFraction=0.5,schedulingPolicy=null,parallelism=1] + + c2[allocFraction=0.7,schedulingPolicy=null,parallelism=1] | trigger trigger_1: if (BYTES_READ > 0) { MOVE TO null_pool } | mapped for groups: group2 + c1[allocFraction=0.3,schedulingPolicy=fair,parallelism=3] @@ -4351,12 +4405,12 @@ POSTHOOK: query: SELECT * FROM SYS.WM_MAPPINGS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_mappings #### A masked pattern was here #### -plan_2 APPLICATION app1 def.c1 0 -plan_2 GROUP group1 def.c1 0 -plan_2 GROUP group2 def.c2 1 -plan_2 GROUP group3 1 -plan_2 USER user1 0 -plan_2 USER user2 def 1 +plan_2 default APPLICATION app1 def.c1 0 +plan_2 default GROUP group1 def.c1 0 +plan_2 default GROUP group2 def.c2 1 +plan_2 default GROUP group3 1 +plan_2 default USER user1 0 +plan_2 default USER user2 def 1 PREHOOK: query: DROP POOL plan_2.def.c1 PREHOOK: type: DROP POOL PREHOOK: Output: dummyHostnameForTest @@ -4389,8 +4443,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_MAPPINGS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_mappings #### A masked pattern was here #### -plan_2 GROUP group1 def.c1 0 -plan_2 USER user1 0 +plan_2 default GROUP group1 def.c1 0 +plan_2 default USER user1 0 PREHOOK: query: CREATE RESOURCE PLAN plan_4 PREHOOK: type: CREATE RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -4409,7 +4463,7 @@ POSTHOOK: type: SHOW RESOURCEPLAN plan_2[status=DISABLED,parallelism=10,defaultPool=def] + def[allocFraction=1.0,schedulingPolicy=null,parallelism=5] | mapped for default - + c2[allocFraction=0.5,schedulingPolicy=null,parallelism=1] + + c2[allocFraction=0.7,schedulingPolicy=null,parallelism=1] | trigger trigger_1: if (BYTES_READ > 0) { MOVE TO null_pool } + c1[allocFraction=0.3,schedulingPolicy=fair,parallelism=3] | mapped for groups: group1 @@ -4433,10 +4487,10 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 ENABLED NULL default -plan_2 DISABLED NULL default -plan_4 ACTIVE NULL default -table DISABLED 1 NULL +plan_1 default ENABLED NULL default +plan_2 default DISABLED NULL default +plan_4 default ACTIVE NULL default +table default DISABLED 1 NULL PREHOOK: query: SELECT * FROM SYS.WM_POOLS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools @@ -4445,11 +4499,11 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 default 1.0 4 NULL -plan_4 default 1.0 4 NULL -table table 0.0 1 fifo -table table.pool 0.9 3 fair +plan_1 default default 1.0 4 NULL +plan_2 default default 1.0 4 NULL +plan_4 default default 1.0 4 NULL +table default table 0.0 1 fifo +table default table.pool 0.9 3 fair PREHOOK: query: SELECT * FROM SYS.WM_TRIGGERS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_triggers @@ -4458,11 +4512,11 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_triggers #### A masked pattern was here #### -plan_1 trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool -table table BYTES_WRITTEN > '100KB' MOVE TO table -table trigger BYTES_WRITTEN > '100MB' MOVE TO default -table trigger1 ELAPSED_TIME > 10 KILL -table trigger2 ELAPSED_TIME > '1hour' KILL +plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool +table default table BYTES_WRITTEN > '100KB' MOVE TO table +table default trigger BYTES_WRITTEN > '100MB' MOVE TO default +table default trigger1 ELAPSED_TIME > 10 KILL +table default trigger2 ELAPSED_TIME > '1hour' KILL PREHOOK: query: SELECT * FROM SYS.WM_POOLS_TO_TRIGGERS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools_to_triggers @@ -4471,7 +4525,7 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS_TO_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools_to_triggers #### A masked pattern was here #### -table table table +table default table table PREHOOK: query: SELECT * FROM SYS.WM_MAPPINGS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_mappings @@ -4528,12 +4582,12 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 ENABLED NULL default -plan_2 DISABLED NULL default -plan_4 ACTIVE NULL default -plan_4a DISABLED NULL default -plan_4b DISABLED NULL default -table DISABLED 1 NULL +plan_1 default ENABLED NULL default +plan_2 default DISABLED NULL default +plan_4 default ACTIVE NULL default +plan_4a default DISABLED NULL default +plan_4b default DISABLED NULL default +table default DISABLED 1 NULL PREHOOK: query: SELECT * FROM SYS.WM_POOLS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools @@ -4542,16 +4596,16 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 default 1.0 4 NULL -plan_4 default 1.0 4 NULL -plan_4a default 1.0 4 NULL -plan_4a pool1 0.0 2 fair -plan_4b default 1.0 4 NULL -plan_4b pool1 0.0 2 fair -plan_4b pool2 0.0 3 fair -table table 0.0 1 fifo -table table.pool 0.9 3 fair +plan_1 default default 1.0 4 NULL +plan_2 default default 1.0 4 NULL +plan_4 default default 1.0 4 NULL +plan_4a default default 1.0 4 NULL +plan_4a default pool1 0.0 2 fair +plan_4b default default 1.0 4 NULL +plan_4b default pool1 0.0 2 fair +plan_4b default pool2 0.0 3 fair +table default table 0.0 1 fifo +table default table.pool 0.9 3 fair PREHOOK: query: SELECT * FROM SYS.WM_TRIGGERS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_triggers @@ -4560,15 +4614,15 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_triggers #### A masked pattern was here #### -plan_1 trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool -plan_4a trigger_1 BYTES_READ > '10GB' KILL -plan_4a trigger_2 BYTES_READ > '11GB' KILL -plan_4b trigger_1 BYTES_READ > '10GB' KILL -plan_4b trigger_2 BYTES_READ > '11GB' KILL -table table BYTES_WRITTEN > '100KB' MOVE TO table -table trigger BYTES_WRITTEN > '100MB' MOVE TO default -table trigger1 ELAPSED_TIME > 10 KILL -table trigger2 ELAPSED_TIME > '1hour' KILL +plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool +plan_4a default trigger_1 BYTES_READ > '10GB' KILL +plan_4a default trigger_2 BYTES_READ > '11GB' KILL +plan_4b default trigger_1 BYTES_READ > '10GB' KILL +plan_4b default trigger_2 BYTES_READ > '11GB' KILL +table default table BYTES_WRITTEN > '100KB' MOVE TO table +table default trigger BYTES_WRITTEN > '100MB' MOVE TO default +table default trigger1 ELAPSED_TIME > 10 KILL +table default trigger2 ELAPSED_TIME > '1hour' KILL PREHOOK: query: SELECT * FROM SYS.WM_POOLS_TO_TRIGGERS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools_to_triggers @@ -4577,9 +4631,9 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS_TO_TRIGGERS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools_to_triggers #### A masked pattern was here #### -plan_4a pool1 trigger_2 -plan_4b pool1 trigger_2 -table table table +plan_4a default pool1 trigger_2 +plan_4b default pool1 trigger_2 +table default table table PREHOOK: query: SELECT * FROM SYS.WM_MAPPINGS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_mappings @@ -4588,8 +4642,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_MAPPINGS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_mappings #### A masked pattern was here #### -plan_4a USER user1 pool1 0 -plan_4b USER user1 pool1 0 +plan_4a default USER user1 pool1 0 +plan_4b default USER user1 pool1 0 PREHOOK: query: REPLACE RESOURCE PLAN plan_4a WITH plan_4b PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -4603,12 +4657,12 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 ENABLED NULL default -plan_2 DISABLED NULL default -plan_4 ACTIVE NULL default -plan_4a DISABLED NULL default -plan_4a_old_0 DISABLED NULL default -table DISABLED 1 NULL +plan_1 default ENABLED NULL default +plan_2 default DISABLED NULL default +plan_4 default ACTIVE NULL default +plan_4a default DISABLED NULL default +plan_4a_old_0 default DISABLED NULL default +table default DISABLED 1 NULL PREHOOK: query: SELECT * FROM SYS.WM_POOLS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools @@ -4617,16 +4671,16 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 default 1.0 4 NULL -plan_4 default 1.0 4 NULL -plan_4a default 1.0 4 NULL -plan_4a pool1 0.0 2 fair -plan_4a pool2 0.0 3 fair -plan_4a_old_0 default 1.0 4 NULL -plan_4a_old_0 pool1 0.0 2 fair -table table 0.0 1 fifo -table table.pool 0.9 3 fair +plan_1 default default 1.0 4 NULL +plan_2 default default 1.0 4 NULL +plan_4 default default 1.0 4 NULL +plan_4a default default 1.0 4 NULL +plan_4a default pool1 0.0 2 fair +plan_4a default pool2 0.0 3 fair +plan_4a_old_0 default default 1.0 4 NULL +plan_4a_old_0 default pool1 0.0 2 fair +table default table 0.0 1 fifo +table default table.pool 0.9 3 fair PREHOOK: query: SHOW RESOURCE PLAN plan_4a_old_0 PREHOOK: type: SHOW RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -4653,12 +4707,12 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 ENABLED NULL default -plan_2 DISABLED NULL default -plan_4 ACTIVE NULL default -plan_4_old_0 DISABLED NULL default -plan_4a_old_0 DISABLED NULL default -table DISABLED 1 NULL +plan_1 default ENABLED NULL default +plan_2 default DISABLED NULL default +plan_4 default ACTIVE NULL default +plan_4_old_0 default DISABLED NULL default +plan_4a_old_0 default DISABLED NULL default +table default DISABLED 1 NULL PREHOOK: query: CREATE RESOURCE PLAN plan_4a LIKE plan_4 PREHOOK: type: CREATE RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest @@ -4682,13 +4736,13 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_1 ENABLED NULL default -plan_2 DISABLED NULL default -plan_4 ACTIVE NULL default -plan_4_old_0 DISABLED NULL default -plan_4_old_1 DISABLED NULL default -plan_4a_old_0 DISABLED NULL default -table DISABLED 1 NULL +plan_1 default ENABLED NULL default +plan_2 default DISABLED NULL default +plan_4 default ACTIVE NULL default +plan_4_old_0 default DISABLED NULL default +plan_4_old_1 default DISABLED NULL default +plan_4a_old_0 default DISABLED NULL default +table default DISABLED 1 NULL PREHOOK: query: SELECT * FROM SYS.WM_POOLS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools @@ -4697,17 +4751,17 @@ POSTHOOK: query: SELECT * FROM SYS.WM_POOLS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_pools #### A masked pattern was here #### -plan_1 default 1.0 4 NULL -plan_2 default 1.0 4 NULL -plan_4 default 1.0 4 NULL -plan_4 pool1 0.0 2 fair -plan_4 pool2 0.0 3 fair -plan_4 pool3 0.0 3 fair -plan_4_old_0 default 1.0 4 NULL -plan_4_old_1 default 1.0 4 NULL -plan_4_old_1 pool1 0.0 2 fair -plan_4_old_1 pool2 0.0 3 fair -plan_4a_old_0 default 1.0 4 NULL -plan_4a_old_0 pool1 0.0 2 fair -table table 0.0 1 fifo -table table.pool 0.9 3 fair +plan_1 default default 1.0 4 NULL +plan_2 default default 1.0 4 NULL +plan_4 default default 1.0 4 NULL +plan_4 default pool1 0.0 2 fair +plan_4 default pool2 0.0 3 fair +plan_4 default pool3 0.0 3 fair +plan_4_old_0 default default 1.0 4 NULL +plan_4_old_1 default default 1.0 4 NULL +plan_4_old_1 default pool1 0.0 2 fair +plan_4_old_1 default pool2 0.0 3 fair +plan_4a_old_0 default default 1.0 4 NULL +plan_4a_old_0 default pool1 0.0 2 fair +table default table 0.0 1 fifo +table default table.pool 0.9 3 fair diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 90b5764a2e24d5ddf71d227f1364a0d09d63c185..a9f11fc073ec40101981db3c1557e164273d40b0 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -51,6 +51,7 @@ import javax.security.auth.login.LoginException; import com.google.common.base.Preconditions; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -3413,34 +3414,39 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName) } @Override - public WMFullResourcePlan getResourcePlan(String resourcePlanName) + public WMFullResourcePlan getResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException { WMGetResourcePlanRequest request = new WMGetResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); return client.get_resource_plan(request).getResourcePlan(); } @Override - public List getAllResourcePlans() + public List getAllResourcePlans(String ns) throws NoSuchObjectException, MetaException, TException { WMGetAllResourcePlanRequest request = new WMGetAllResourcePlanRequest(); + request.setNs(ns); return client.get_all_resource_plans(request).getResourcePlans(); } @Override - public void dropResourcePlan(String resourcePlanName) + public void dropResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException { WMDropResourcePlanRequest request = new WMDropResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); client.drop_resource_plan(request); } @Override - public WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String resourcePlanName, String ns, + WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) throws NoSuchObjectException, InvalidObjectException, MetaException, TException { WMAlterResourcePlanRequest request = new WMAlterResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); request.setResourcePlan(resourcePlan); request.setIsEnableAndActivate(canActivateDisabled); request.setIsForceDeactivate(isForceDeactivate); @@ -3450,15 +3456,18 @@ public WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableR } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException, TException { - return client.get_active_resource_plan(new WMGetActiveResourcePlanRequest()).getResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException, TException { + WMGetActiveResourcePlanRequest request = new WMGetActiveResourcePlanRequest(); + request.setNs(ns); + return client.get_active_resource_plan(request).getResourcePlan(); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName) + public WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException, TException { WMValidateResourcePlanRequest request = new WMValidateResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); return client.validate_resource_plan(request); } @@ -3479,19 +3488,21 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, MetaException, TException { WMDropTriggerRequest request = new WMDropTriggerRequest(); request.setResourcePlanName(resourcePlanName); request.setTriggerName(triggerName); + request.setNs(ns); client.drop_wm_trigger(request); } @Override - public List getTriggersForResourcePlan(String resourcePlan) + public List getTriggersForResourcePlan(String resourcePlan, String ns) throws NoSuchObjectException, MetaException, TException { WMGetTriggersForResourePlanRequest request = new WMGetTriggersForResourePlanRequest(); request.setResourcePlanName(resourcePlan); + request.setNs(ns); return client.get_triggers_for_resourceplan(request).getTriggers(); } @@ -3513,11 +3524,12 @@ public void alterWMPool(WMNullablePool pool, String poolPath) } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, MetaException, TException { WMDropPoolRequest request = new WMDropPoolRequest(); request.setResourcePlanName(resourcePlanName); request.setPoolPath(poolPath); + request.setNs(ns); client.drop_wm_pool(request); } @@ -3540,13 +3552,14 @@ public void dropWMMapping(WMMapping mapping) @Override public void createOrDropTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath, boolean shouldDrop) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, boolean shouldDrop, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException { WMCreateOrDropTriggerToPoolMappingRequest request = new WMCreateOrDropTriggerToPoolMappingRequest(); request.setResourcePlanName(resourcePlanName); request.setTriggerName(triggerName); request.setPoolPath(poolPath); request.setDrop(shouldDrop); + request.setNs(ns); client.create_or_drop_wm_trigger_to_pool_mapping(request); } diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index d1c0c4d1f60016f28cea69348b1b30ecb61bf083..fa19440ba29446bffb00de1ae0cf5e64fbcd4d12 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -3542,22 +3542,22 @@ void addCheckConstraint(List checkConstraints) throws void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName) throws InvalidObjectException, MetaException, TException; - WMFullResourcePlan getResourcePlan(String resourcePlanName) + WMFullResourcePlan getResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException; - List getAllResourcePlans() + List getAllResourcePlans(String ns) throws NoSuchObjectException, MetaException, TException; - void dropResourcePlan(String resourcePlanName) + void dropResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException; - WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableResourcePlan resourcePlan, + WMFullResourcePlan alterResourcePlan(String resourcePlanName, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) throws NoSuchObjectException, InvalidObjectException, MetaException, TException; - WMFullResourcePlan getActiveResourcePlan() throws MetaException, TException; + WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException, TException; - WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName) + WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException, TException; void createWMTrigger(WMTrigger trigger) @@ -3566,10 +3566,10 @@ void createWMTrigger(WMTrigger trigger) void alterWMTrigger(WMTrigger trigger) throws NoSuchObjectException, InvalidObjectException, MetaException, TException; - void dropWMTrigger(String resourcePlanName, String triggerName) + void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, MetaException, TException; - List getTriggersForResourcePlan(String resourcePlan) + List getTriggersForResourcePlan(String resourcePlan, String ns) throws NoSuchObjectException, MetaException, TException; void createWMPool(WMPool pool) @@ -3578,7 +3578,7 @@ void createWMPool(WMPool pool) void alterWMPool(WMNullablePool pool, String poolPath) throws NoSuchObjectException, InvalidObjectException, TException; - void dropWMPool(String resourcePlanName, String poolPath) + void dropWMPool(String resourcePlanName, String poolPath, String ns) throws TException; void createOrUpdateWMMapping(WMMapping mapping, boolean isUpdate) @@ -3588,7 +3588,7 @@ void dropWMMapping(WMMapping mapping) throws TException; void createOrDropTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath, boolean shouldDrop) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, boolean shouldDrop, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException; /** diff --git standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 4b7b61520a2d55635f474317053a17410f3a4bb7..48b633e5fae02db6e6d20f9121126ef147726566 100644 --- standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -1343,6 +1343,7 @@ struct WMResourcePlan { 2: optional WMResourcePlanStatus status; 3: optional i32 queryParallelism; 4: optional string defaultPoolPath; + 5: optional string ns; } struct WMNullableResourcePlan { @@ -1352,6 +1353,7 @@ struct WMNullableResourcePlan { 5: optional bool isSetQueryParallelism; 6: optional string defaultPoolPath; 7: optional bool isSetDefaultPoolPath; + 8: optional string ns; } struct WMPool { @@ -1360,6 +1362,7 @@ struct WMPool { 3: optional double allocFraction; 4: optional i32 queryParallelism; 5: optional string schedulingPolicy; + 6: optional string ns; } @@ -1370,6 +1373,7 @@ struct WMNullablePool { 4: optional i32 queryParallelism; 5: optional string schedulingPolicy; 6: optional bool isSetSchedulingPolicy; + 7: optional string ns; } struct WMTrigger { @@ -1378,6 +1382,7 @@ struct WMTrigger { 3: optional string triggerExpression; 4: optional string actionExpression; 5: optional bool isInUnmanaged; + 6: optional string ns; } struct WMMapping { @@ -1386,11 +1391,13 @@ struct WMMapping { 3: required string entityName; 4: optional string poolPath; 5: optional i32 ordering; + 6: optional string ns; } struct WMPoolTrigger { 1: required string pool; 2: required string trigger; + 3: optional string ns; } struct WMFullResourcePlan { @@ -1412,6 +1419,7 @@ struct WMCreateResourcePlanResponse { } struct WMGetActiveResourcePlanRequest { + 1: optional string ns; } struct WMGetActiveResourcePlanResponse { @@ -1420,6 +1428,7 @@ struct WMGetActiveResourcePlanResponse { struct WMGetResourcePlanRequest { 1: optional string resourcePlanName; + 2: optional string ns; } struct WMGetResourcePlanResponse { @@ -1427,6 +1436,7 @@ struct WMGetResourcePlanResponse { } struct WMGetAllResourcePlanRequest { + 1: optional string ns; } struct WMGetAllResourcePlanResponse { @@ -1439,6 +1449,7 @@ struct WMAlterResourcePlanRequest { 3: optional bool isEnableAndActivate; 4: optional bool isForceDeactivate; 5: optional bool isReplace; + 6: optional string ns; } struct WMAlterResourcePlanResponse { @@ -1447,6 +1458,7 @@ struct WMAlterResourcePlanResponse { struct WMValidateResourcePlanRequest { 1: optional string resourcePlanName; + 2: optional string ns; } struct WMValidateResourcePlanResponse { @@ -1456,6 +1468,7 @@ struct WMValidateResourcePlanResponse { struct WMDropResourcePlanRequest { 1: optional string resourcePlanName; + 2: optional string ns; } struct WMDropResourcePlanResponse { @@ -1478,6 +1491,7 @@ struct WMAlterTriggerResponse { struct WMDropTriggerRequest { 1: optional string resourcePlanName; 2: optional string triggerName; + 3: optional string ns; } struct WMDropTriggerResponse { @@ -1485,6 +1499,7 @@ struct WMDropTriggerResponse { struct WMGetTriggersForResourePlanRequest { 1: optional string resourcePlanName; + 2: optional string ns; } struct WMGetTriggersForResourePlanResponse { @@ -1509,6 +1524,7 @@ struct WMAlterPoolResponse { struct WMDropPoolRequest { 1: optional string resourcePlanName; 2: optional string poolPath; + 3: optional string ns; } struct WMDropPoolResponse { @@ -1534,6 +1550,7 @@ struct WMCreateOrDropTriggerToPoolMappingRequest { 2: optional string triggerName; 3: optional string poolPath; 4: optional bool drop; + 5: optional string ns; } struct WMCreateOrDropTriggerToPoolMappingResponse { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 8cd46e3f44e7c4e47fbf7f2ce2b6350a5814106f..04851845543d326407a04d33af611e00c9f2c76f 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -8200,7 +8200,7 @@ public WMCreateResourcePlanResponse create_resource_plan(WMCreateResourcePlanReq public WMGetResourcePlanResponse get_resource_plan(WMGetResourcePlanRequest request) throws NoSuchObjectException, MetaException, TException { try { - WMFullResourcePlan rp = getMS().getResourcePlan(request.getResourcePlanName()); + WMFullResourcePlan rp = getMS().getResourcePlan(request.getResourcePlanName(), request.getNs()); WMGetResourcePlanResponse resp = new WMGetResourcePlanResponse(); resp.setResourcePlan(rp); return resp; @@ -8215,7 +8215,7 @@ public WMGetAllResourcePlanResponse get_all_resource_plans(WMGetAllResourcePlanR throws MetaException, TException { try { WMGetAllResourcePlanResponse resp = new WMGetAllResourcePlanResponse(); - resp.setResourcePlans(getMS().getAllResourcePlans()); + resp.setResourcePlans(getMS().getAllResourcePlans(request.getNs())); return resp; } catch (MetaException e) { LOG.error("Exception while trying to retrieve resource plans", e); @@ -8235,7 +8235,7 @@ public WMAlterResourcePlanResponse alter_resource_plan(WMAlterResourcePlanReques // This method will only return full resource plan when activating one, // to give the caller the result atomically with the activation. WMFullResourcePlan fullPlanAfterAlter = getMS().alterResourcePlan( - request.getResourcePlanName(), request.getResourcePlan(), + request.getResourcePlanName(), request.getNs(), request.getResourcePlan(), request.isIsEnableAndActivate(), request.isIsForceDeactivate(), request.isIsReplace()); if (fullPlanAfterAlter != null) { response.setFullResourcePlan(fullPlanAfterAlter); @@ -8252,7 +8252,7 @@ public WMGetActiveResourcePlanResponse get_active_resource_plan( WMGetActiveResourcePlanRequest request) throws MetaException, TException { try { WMGetActiveResourcePlanResponse response = new WMGetActiveResourcePlanResponse(); - response.setResourcePlan(getMS().getActiveResourcePlan()); + response.setResourcePlan(getMS().getActiveResourcePlan(request.getNs())); return response; } catch (MetaException e) { LOG.error("Exception while trying to get active resource plan", e); @@ -8264,7 +8264,7 @@ public WMGetActiveResourcePlanResponse get_active_resource_plan( public WMValidateResourcePlanResponse validate_resource_plan(WMValidateResourcePlanRequest request) throws NoSuchObjectException, MetaException, TException { try { - return getMS().validateResourcePlan(request.getResourcePlanName()); + return getMS().validateResourcePlan(request.getResourcePlanName(), request.getNs()); } catch (MetaException e) { LOG.error("Exception while trying to validate resource plan", e); throw e; @@ -8275,7 +8275,7 @@ public WMValidateResourcePlanResponse validate_resource_plan(WMValidateResourceP public WMDropResourcePlanResponse drop_resource_plan(WMDropResourcePlanRequest request) throws NoSuchObjectException, InvalidOperationException, MetaException, TException { try { - getMS().dropResourcePlan(request.getResourcePlanName()); + getMS().dropResourcePlan(request.getResourcePlanName(), request.getNs()); return new WMDropResourcePlanResponse(); } catch (MetaException e) { LOG.error("Exception while trying to drop resource plan", e); @@ -8311,7 +8311,7 @@ public WMAlterTriggerResponse alter_wm_trigger(WMAlterTriggerRequest request) public WMDropTriggerResponse drop_wm_trigger(WMDropTriggerRequest request) throws NoSuchObjectException, InvalidOperationException, MetaException, TException { try { - getMS().dropWMTrigger(request.getResourcePlanName(), request.getTriggerName()); + getMS().dropWMTrigger(request.getResourcePlanName(), request.getTriggerName(), request.getNs()); return new WMDropTriggerResponse(); } catch (MetaException e) { LOG.error("Exception while trying to drop trigger.", e); @@ -8325,7 +8325,7 @@ public WMGetTriggersForResourePlanResponse get_triggers_for_resourceplan( throws NoSuchObjectException, MetaException, TException { try { List triggers = - getMS().getTriggersForResourcePlan(request.getResourcePlanName()); + getMS().getTriggersForResourcePlan(request.getResourcePlanName(), request.getNs()); WMGetTriggersForResourePlanResponse response = new WMGetTriggersForResourePlanResponse(); response.setTriggers(triggers); return response; @@ -8365,7 +8365,7 @@ public WMCreatePoolResponse create_wm_pool(WMCreatePoolRequest request) public WMDropPoolResponse drop_wm_pool(WMDropPoolRequest request) throws NoSuchObjectException, InvalidOperationException, MetaException, TException { try { - getMS().dropWMPool(request.getResourcePlanName(), request.getPoolPath()); + getMS().dropWMPool(request.getResourcePlanName(), request.getPoolPath(), request.getNs()); return new WMDropPoolResponse(); } catch (MetaException e) { LOG.error("Exception while trying to drop WMPool", e); @@ -8404,11 +8404,11 @@ public WMCreateOrDropTriggerToPoolMappingResponse create_or_drop_wm_trigger_to_p NoSuchObjectException, InvalidObjectException, MetaException, TException { try { if (request.isDrop()) { - getMS().dropWMTriggerToPoolMapping( - request.getResourcePlanName(), request.getTriggerName(), request.getPoolPath()); + getMS().dropWMTriggerToPoolMapping(request.getResourcePlanName(), + request.getTriggerName(), request.getPoolPath(), request.getNs()); } else { - getMS().createWMTriggerToPoolMapping( - request.getResourcePlanName(), request.getTriggerName(), request.getPoolPath()); + getMS().createWMTriggerToPoolMapping(request.getResourcePlanName(), + request.getTriggerName(), request.getPoolPath(), request.getNs()); } return new WMCreateOrDropTriggerToPoolMappingResponse(); } catch (MetaException e) { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 9c158040497cd3d2762620ce35e2b46bb6d5fffe..366fbb66d3b9a410ea4879a619b2e3d0ab3675e1 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -11593,13 +11593,15 @@ public void createResourcePlan( } else { rp = new MWMResourcePlan(rpName, null, Status.DISABLED); } + rp.setNs(resourcePlan.getNs()); try { openTransaction(); pm.makePersistent(rp); if (copyFromName != null) { - MWMResourcePlan copyFrom = getMWMResourcePlan(copyFromName, false); + String ns = getNsOrDefault(resourcePlan.getNs()); + MWMResourcePlan copyFrom = getMWMResourcePlan(copyFromName, ns, false); if (copyFrom == null) { - throw new NoSuchObjectException(copyFromName); + throw new NoSuchObjectException(copyFromName + " in " + ns); } copyRpContents(rp, copyFrom); } else { @@ -11627,6 +11629,7 @@ public void createResourcePlan( private void copyRpContents(MWMResourcePlan dest, MWMResourcePlan src) { dest.setQueryParallelism(src.getQueryParallelism()); + dest.setNs(src.getNs()); Map pools = new HashMap<>(); Map> triggersToPools = new HashMap<>(); for (MWMPool copyPool : src.getPools()) { @@ -11686,6 +11689,7 @@ private WMResourcePlan fromMResourcePlan(MWMResourcePlan mplan) { } WMResourcePlan rp = new WMResourcePlan(); rp.setName(mplan.getName()); + rp.setNs(mplan.getNs()); rp.setStatus(WMResourcePlanStatus.valueOf(mplan.getStatus().name())); if (mplan.getQueryParallelism() != null) { rp.setQueryParallelism(mplan.getQueryParallelism()); @@ -11724,6 +11728,7 @@ private WMPool fromMPool(MWMPool mPool, String rpName) { assert mPool.getQueryParallelism() != null; result.setQueryParallelism(mPool.getQueryParallelism()); result.setSchedulingPolicy(mPool.getSchedulingPolicy()); + result.setNs(mPool.getResourcePlan().getNs()); return result; } @@ -11736,15 +11741,24 @@ private WMMapping fromMMapping(MWMMapping mMapping, String rpName) { if (mMapping.getOrdering() != null) { result.setOrdering(mMapping.getOrdering()); } + result.setNs(mMapping.getResourcePlan().getNs()); return result; } + private final String getNsOrDefault(String ns) { + // This is only needed for old clients not setting NS in requests. + // Not clear how to handle this... this is properly a HS2 config but metastore needs its default + // value for backward compat, and we don't want it configurable separately because it's also + // used in upgrade scripts, were it cannot be configured. + return normalizeIdentifier(ns == null ? "default" : ns); + } + @Override - public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException { + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException { boolean commited = false; try { openTransaction(); - WMFullResourcePlan fullRp = fullFromMResourcePlan(getMWMResourcePlan(name, false)); + WMFullResourcePlan fullRp = fullFromMResourcePlan(getMWMResourcePlan(name, ns, false)); commited = commitTransaction(); return fullRp; } catch (InvalidOperationException e) { @@ -11755,12 +11769,12 @@ public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectExcept } } - private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck) + private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck) throws NoSuchObjectException, InvalidOperationException { - return getMWMResourcePlan(name, editCheck, true); + return getMWMResourcePlan(name, ns, editCheck, true); } - private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck, boolean mustExist) + private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck, boolean mustExist) throws NoSuchObjectException, InvalidOperationException { MWMResourcePlan resourcePlan; boolean commited = false; @@ -11768,18 +11782,16 @@ private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck, boole name = normalizeIdentifier(name); try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "name == rpname"); - query.declareParameters("java.lang.String rpname"); - query.setUnique(true); - resourcePlan = (MWMResourcePlan) query.execute(name); + query = createGetResourcePlanQuery(); + ns = getNsOrDefault(ns); + resourcePlan = (MWMResourcePlan) query.execute(name, ns); pm.retrieve(resourcePlan); commited = commitTransaction(); } finally { rollbackAndCleanup(commited, query); } if (mustExist && resourcePlan == null) { - throw new NoSuchObjectException("There is no resource plan named: " + name); + throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); } if (editCheck && resourcePlan != null && resourcePlan.getStatus() != MWMResourcePlan.Status.DISABLED) { @@ -11788,15 +11800,24 @@ private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck, boole return resourcePlan; } + private Query createGetResourcePlanQuery() { + openTransaction(); + Query query = pm.newQuery(MWMResourcePlan.class, "name == rpname && ns == nsname"); + query.declareParameters("java.lang.String rpname, java.lang.String nsname"); + query.setUnique(true); + return query; + } + @Override - public List getAllResourcePlans() throws MetaException { + public List getAllResourcePlans(String ns) throws MetaException { List resourcePlans = new ArrayList(); boolean commited = false; Query query = null; try { openTransaction(); - query = pm.newQuery(MWMResourcePlan.class); - List mplans = (List) query.execute(); + query = pm.newQuery(MWMResourcePlan.class, "ns == nsname"); + query.declareParameters("java.lang.String nsname"); + List mplans = (List) query.execute(getNsOrDefault(ns)); pm.retrieveAll(mplans); commited = commitTransaction(); if (mplans != null) { @@ -11811,7 +11832,7 @@ private MWMResourcePlan getMWMResourcePlan(String name, boolean editCheck, boole } @Override - public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan changes, + public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan changes, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { name = name == null ? null : normalizeIdentifier(name); @@ -11827,9 +11848,9 @@ public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan try { openTransaction(); if (isReplace) { - result = handleAlterReplace(name, changes); + result = handleAlterReplace(name, ns, changes); } else { - result = handleSimpleAlter(name, changes, canActivateDisabled, canDeactivate); + result = handleSimpleAlter(name, ns, changes, canActivateDisabled, canDeactivate); } commited = commitTransaction(); @@ -11842,11 +11863,16 @@ public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan } } - private WMFullResourcePlan handleSimpleAlter(String name, WMNullableResourcePlan changes, + private WMFullResourcePlan handleSimpleAlter(String name, String ns, WMNullableResourcePlan changes, boolean canActivateDisabled, boolean canDeactivate) throws InvalidOperationException, NoSuchObjectException, MetaException { - MWMResourcePlan plan = name == null ? getActiveMWMResourcePlan() - : getMWMResourcePlan(name, !changes.isSetStatus()); + MWMResourcePlan plan = name == null ? getActiveMWMResourcePlan(ns) + : getMWMResourcePlan(name, ns, !changes.isSetStatus()); + boolean hasNsChange = changes.isSetNs() && !changes.getNs().equals(getNsOrDefault(plan.getNs())); + if (hasNsChange) { + throw new InvalidOperationException("Cannot change ns; from " + getNsOrDefault(plan.getNs()) + + " to " + changes.getNs()); + } boolean hasNameChange = changes.isSetName() && !changes.getName().equals(name); // Verify that field changes are consistent with what Hive does. Note: we could handle this. if (changes.isSetIsSetQueryParallelism() @@ -11895,7 +11921,7 @@ private WMFullResourcePlan handleSimpleAlter(String name, WMNullableResourcePlan return null; } - private WMFullResourcePlan handleAlterReplace(String name, WMNullableResourcePlan changes) + private WMFullResourcePlan handleAlterReplace(String name, String ns, WMNullableResourcePlan changes) throws InvalidOperationException, NoSuchObjectException, MetaException { // Verify that field changes are consistent with what Hive does. Note: we could handle this. if (changes.isSetQueryParallelism() || changes.isSetDefaultPoolPath()) { @@ -11911,19 +11937,26 @@ private WMFullResourcePlan handleAlterReplace(String name, WMNullableResourcePla if (name == null) { throw new InvalidOperationException("Invalid replace - no name specified"); } + ns = getNsOrDefault(ns); MWMResourcePlan replacedPlan = isReplacingSpecific - ? getMWMResourcePlan(changes.getName(), false) : getActiveMWMResourcePlan(); - MWMResourcePlan plan = getMWMResourcePlan(name, false); + ? getMWMResourcePlan(changes.getName(), ns, false) : getActiveMWMResourcePlan(ns); + MWMResourcePlan plan = getMWMResourcePlan(name, ns, false); if (replacedPlan.getName().equals(plan.getName())) { throw new InvalidOperationException("A plan cannot replace itself"); } + String oldNs = getNsOrDefault(replacedPlan.getNs()), newNs = getNsOrDefault(plan.getNs()); + if (!oldNs.equals(newNs)) { + throw new InvalidOperationException("Cannot change the namespace; replacing " + + oldNs + " with " + newNs); + } + // We will inherit the name and status from the plan we are replacing. String newName = replacedPlan.getName(); int i = 0; String copyName = generateOldPlanName(newName, i); while (true) { - MWMResourcePlan dup = getMWMResourcePlan(copyName, false, false); + MWMResourcePlan dup = getMWMResourcePlan(copyName, ns, false, false); if (dup == null) { break; } @@ -11950,18 +11983,16 @@ private String generateOldPlanName(String newName, int i) { } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { // Note: fullFromMResroucePlan needs to be called inside the txn, otherwise we could have // deduplicated this with getActiveMWMResourcePlan. boolean commited = false; Query query = null; WMFullResourcePlan result = null; try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "status == activeStatus"); - query.declareParameters("java.lang.String activeStatus"); - query.setUnique(true); - MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(Status.ACTIVE.toString()); + query = createActivePlanQuery(); + MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute( + Status.ACTIVE.toString(), getNsOrDefault(ns)); if (mResourcePlan != null) { result = fullFromMResourcePlan(mResourcePlan); } @@ -11972,16 +12003,14 @@ public WMFullResourcePlan getActiveResourcePlan() throws MetaException { return result; } - private MWMResourcePlan getActiveMWMResourcePlan() throws MetaException { + private MWMResourcePlan getActiveMWMResourcePlan(String ns) throws MetaException { boolean commited = false; Query query = null; MWMResourcePlan result = null; try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "status == activeStatus"); - query.declareParameters("java.lang.String activeStatus"); - query.setUnique(true); - result = (MWMResourcePlan) query.execute(Status.ACTIVE.toString()); + query = createActivePlanQuery(); + result = (MWMResourcePlan) query.execute( + Status.ACTIVE.toString(), getNsOrDefault(ns)); pm.retrieve(result); commited = commitTransaction(); } finally { @@ -11990,6 +12019,14 @@ private MWMResourcePlan getActiveMWMResourcePlan() throws MetaException { return result; } + private Query createActivePlanQuery() { + openTransaction(); + Query query = pm.newQuery(MWMResourcePlan.class, "status == activeStatus && ns == nsname"); + query.declareParameters("java.lang.String activeStatus, java.lang.String nsname"); + query.setUnique(true); + return query; + } + private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePlan, String status, boolean canActivateDisabled, boolean canDeactivate) throws InvalidOperationException { Status currentStatus = mResourcePlan.getStatus(); @@ -12018,7 +12055,7 @@ private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePl doValidate = true; doActivate = (newStatus == Status.ACTIVE); if (doActivate && !canActivateDisabled) { - throw new InvalidOperationException("Resource plan " +name + throw new InvalidOperationException("Resource plan " + name + " is disabled and should be enabled before activation (or in the same command)"); } break; @@ -12043,7 +12080,7 @@ private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePl } if (doActivate) { // Deactivate currently active resource plan. - deactivateActiveResourcePlan(); + deactivateActiveResourcePlan(mResourcePlan.getNs()); mResourcePlan.setStatus(newStatus); return fullFromMResourcePlan(mResourcePlan); } else { @@ -12052,14 +12089,13 @@ private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePl return null; } - private void deactivateActiveResourcePlan() { + private void deactivateActiveResourcePlan(String ns) { boolean commited = false; Query query = null; try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "status == \"ACTIVE\""); - query.setUnique(true); - MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(); + query = createActivePlanQuery(); + MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute( + Status.ACTIVE.toString(), getNsOrDefault(ns)); // We may not have an active resource plan in the start. if (mResourcePlan != null) { mResourcePlan.setStatus(Status.ENABLED); @@ -12130,7 +12166,8 @@ private WMValidateResourcePlanResponse getResourcePlanErrors(MWMResourcePlan mRe poolData.found = true; if (!poolData.hasChildren) { response.addToErrors("Root has no children"); - } else if (Math.abs(1.0 - poolData.totalChildrenAllocFraction) > 0.001) { + // TODO: change fractions to use decimal? somewhat brittle + } else if (Math.abs(1.0 - poolData.totalChildrenAllocFraction) > 0.00001) { response.addToErrors("Sum of root children pools' alloc fraction should be 1.0 got: " + poolData.totalChildrenAllocFraction + " for pool: " + entry.getKey()); } @@ -12139,7 +12176,8 @@ private WMValidateResourcePlanResponse getResourcePlanErrors(MWMResourcePlan mRe response.addToErrors("Pool does not exists but has children: " + entry.getKey()); } if (poolData.hasChildren) { - if (!isRoot && 1.0 <= poolData.totalChildrenAllocFraction) { + + if (!isRoot && (poolData.totalChildrenAllocFraction - 1.0) > 0.00001) { response.addToErrors("Sum of children pools' alloc fraction should be less than 1 got: " + poolData.totalChildrenAllocFraction + " for pool: " + entry.getKey()); } @@ -12151,39 +12189,36 @@ private WMValidateResourcePlanResponse getResourcePlanErrors(MWMResourcePlan mRe } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { name = normalizeIdentifier(name); + boolean committed = false; Query query = null; try { - query = pm.newQuery(MWMResourcePlan.class, "name == rpName"); - query.declareParameters("java.lang.String rpName"); - query.setUnique(true); - MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(name); + query = createGetResourcePlanQuery(); + MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(name, ns); if (mResourcePlan == null) { - throw new NoSuchObjectException("Cannot find resourcePlan: " + name); + throw new NoSuchObjectException("Cannot find resourcePlan: " + name + " in " + ns); } - // Validate resource plan. - return getResourcePlanErrors(mResourcePlan); + WMValidateResourcePlanResponse result = getResourcePlanErrors(mResourcePlan); + committed = commitTransaction(); + return result; } finally { - rollbackAndCleanup(true, query); + rollbackAndCleanup(committed, query); } } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { name = normalizeIdentifier(name); boolean commited = false; Query query = null; try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "name == rpname"); - query.declareParameters("java.lang.String rpname"); - query.setUnique(true); - MWMResourcePlan resourcePlan = (MWMResourcePlan) query.execute(name); - pm.retrieve(resourcePlan); + query = createGetResourcePlanQuery(); + MWMResourcePlan resourcePlan = (MWMResourcePlan) query.execute(name, getNsOrDefault(ns)); + pm.retrieve(resourcePlan); // TODO: why do some codepaths call retrieve and some don't? if (resourcePlan == null) { - throw new NoSuchObjectException("There is no resource plan named: " + name); + throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); } if (resourcePlan.getStatus() == Status.ACTIVE) { throw new MetaException("Cannot drop an active resource plan"); @@ -12207,7 +12242,8 @@ public void createWMTrigger(WMTrigger trigger) boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(trigger.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + trigger.getResourcePlanName(), trigger.getNs(), true); MWMTrigger mTrigger = new MWMTrigger(resourcePlan, normalizeIdentifier(trigger.getTriggerName()), trigger.getTriggerExpression(), trigger.getActionExpression(), null, @@ -12229,7 +12265,8 @@ public void alterWMTrigger(WMTrigger trigger) Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(trigger.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + trigger.getResourcePlanName(), trigger.getNs(), true); MWMTrigger mTrigger = getTrigger(resourcePlan, trigger.getTriggerName()); // Update the object. if (trigger.isSetTriggerExpression()) { @@ -12271,7 +12308,7 @@ private MWMTrigger getTrigger(MWMResourcePlan resourcePlan, String triggerName) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { resourcePlanName = normalizeIdentifier(resourcePlanName); triggerName = normalizeIdentifier(triggerName); @@ -12280,7 +12317,7 @@ public void dropWMTrigger(String resourcePlanName, String triggerName) Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, true); + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp && name == triggerName"); query.declareParameters("MWMResourcePlan rp, java.lang.String triggerName"); if (query.deletePersistentAll(resourcePlan, triggerName) != 1) { @@ -12293,7 +12330,7 @@ public void dropWMTrigger(String resourcePlanName, String triggerName) } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { List triggers = new ArrayList(); boolean commited = false; @@ -12302,7 +12339,7 @@ public void dropWMTrigger(String resourcePlanName, String triggerName) openTransaction(); MWMResourcePlan resourcePlan; try { - resourcePlan = getMWMResourcePlan(resourcePlanName, false); + resourcePlan = getMWMResourcePlan(resourcePlanName, ns, false); } catch (InvalidOperationException e) { // Should not happen, edit check is false. throw new RuntimeException(e); @@ -12330,6 +12367,7 @@ private WMTrigger fromMWMTrigger(MWMTrigger mTrigger, String resourcePlanName) { trigger.setTriggerExpression(mTrigger.getTriggerExpression()); trigger.setActionExpression(mTrigger.getActionExpression()); trigger.setIsInUnmanaged(mTrigger.getIsInUnmanaged()); + trigger.setNs(mTrigger.getResourcePlan().getNs()); return trigger; } @@ -12339,7 +12377,8 @@ public void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectE boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(pool.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + pool.getResourcePlanName(), pool.getNs(), true); if (!poolParentExists(resourcePlan, pool.getPoolPath())) { throw new NoSuchObjectException("Pool path is invalid, the parent does not exist"); @@ -12366,7 +12405,8 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(pool.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + pool.getResourcePlanName(), pool.getNs(), true); MWMPool mPool = getPool(resourcePlan, poolPath); pm.retrieve(mPool); if (pool.isSetAllocFraction()) { @@ -12463,20 +12503,20 @@ private boolean poolParentExists(MWMResourcePlan resourcePlan, String poolPath) } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { poolPath = normalizeIdentifier(poolPath); boolean commited = false; Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, true); + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); if (resourcePlan.getDefaultPool() != null && resourcePlan.getDefaultPool().getPath().equals(poolPath)) { throw new InvalidOperationException("Cannot drop default pool of a resource plan"); } if (poolHasChildren(resourcePlan, poolPath)) { - throw new InvalidOperationException("Pool has children cannot drop."); + throw new InvalidOperationException("Cannot drop a pool that has child pools"); } query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)"); query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); @@ -12521,7 +12561,8 @@ public void createOrUpdateWMMapping(WMMapping mapping, boolean update) Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(mapping.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + mapping.getResourcePlanName(), mapping.getNs(), true); MWMPool pool = null; if (mapping.isSetPoolPath()) { pool = getPool(resourcePlan, mapping.getPoolPath()); @@ -12555,7 +12596,8 @@ public void dropWMMapping(WMMapping mapping) Query query = null; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(mapping.getResourcePlanName(), true); + MWMResourcePlan resourcePlan = getMWMResourcePlan( + mapping.getResourcePlanName(), mapping.getNs(), true); query = pm.newQuery(MWMMapping.class, "resourcePlan == rp && entityType == type && entityName == name"); query.declareParameters("MWMResourcePlan rp, java.lang.String type, java.lang.String name"); @@ -12570,12 +12612,12 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, true); + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); MWMPool pool = getPool(resourcePlan, poolPath); MWMTrigger trigger = getTrigger(resourcePlan, triggerName); pool.getTriggers().add(trigger); @@ -12590,11 +12632,11 @@ public void createWMTriggerToPoolMapping(String resourcePlanName, String trigger @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { boolean commited = false; try { openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, true); + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); MWMPool pool = getPool(resourcePlan, poolPath); MWMTrigger trigger = getTrigger(resourcePlan, triggerName); pool.getTriggers().remove(trigger); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index c3914b668fac18ead6196a4fc449e909f5af01b1..209e92a415c7698c6ce4c4b5a3884aa61b8bb833 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -20,12 +20,6 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.*; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; -import org.apache.hadoop.hive.metastore.api.ISchemaName; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; -import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; -import org.apache.hadoop.hive.metastore.api.WriteEventInfo; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -1546,21 +1540,21 @@ void dropConstraint(String catName, String dbName, String tableName, String cons void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int defaultPoolSize) throws AlreadyExistsException, MetaException, InvalidObjectException, NoSuchObjectException; - WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException, MetaException; + WMFullResourcePlan getResourcePlan(String name, String string) throws NoSuchObjectException, MetaException; - List getAllResourcePlans() throws MetaException; + List getAllResourcePlans(String string) throws MetaException; - WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, + WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException; - WMFullResourcePlan getActiveResourcePlan() throws MetaException; + WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException; - WMValidateResourcePlanResponse validateResourcePlan(String name) + WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException; - void dropResourcePlan(String name) throws NoSuchObjectException, MetaException; + void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException; void createWMTrigger(WMTrigger trigger) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, @@ -1569,10 +1563,10 @@ void createWMTrigger(WMTrigger trigger) void alterWMTrigger(WMTrigger trigger) throws NoSuchObjectException, InvalidOperationException, MetaException; - void dropWMTrigger(String resourcePlanName, String triggerName) + void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException; - List getTriggersForResourcePlan(String resourcePlanName) + List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException; void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectException, @@ -1581,7 +1575,7 @@ void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectExceptio void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException; - void dropWMPool(String resourcePlanName, String poolPath) + void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException; void createOrUpdateWMMapping(WMMapping mapping, boolean update) @@ -1591,11 +1585,11 @@ void createOrUpdateWMMapping(WMMapping mapping, boolean update) void dropWMMapping(WMMapping mapping) throws NoSuchObjectException, InvalidOperationException, MetaException; - void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath) + void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException; - void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath) + void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException; /** diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 47ac68c667bea8f09f5301a6364c854bc18b3c0d..e4ef46fdb4be4dea25bc8e0839e159413223fd35 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -1644,7 +1644,7 @@ public Partition getPartitionWithAuth(String catName, String dbName, String tblN } return partitions; } - + private String getPartNameMatcher(Table table, List partSpecs) throws MetaException { List partCols = table.getPartitionKeys(); int numPartKeys = partCols.size(); @@ -2407,39 +2407,39 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int } @Override - public WMFullResourcePlan getResourcePlan(String name) + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { - return rawStore.getResourcePlan(name); + return rawStore.getResourcePlan(name, ns); } @Override - public List getAllResourcePlans() throws MetaException { - return rawStore.getAllResourcePlans(); + public List getAllResourcePlans(String ns) throws MetaException { + return rawStore.getAllResourcePlans(ns); } @Override - public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { return rawStore.alterResourcePlan( - name, resourcePlan, canActivateDisabled, canDeactivate, isReplace); + name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { - return rawStore.getActiveResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { + return rawStore.getActiveResourcePlan(ns); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { - return rawStore.validateResourcePlan(name); + return rawStore.validateResourcePlan(name, ns); } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { - rawStore.dropResourcePlan(name); + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + rawStore.dropResourcePlan(name, ns); } @Override @@ -2456,15 +2456,15 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMTrigger(resourcePlanName, triggerName); + rawStore.dropWMTrigger(resourcePlanName, triggerName, ns); } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { - return rawStore.getTriggersForResourcePlan(resourcePlanName); + return rawStore.getTriggersForResourcePlan(resourcePlanName, ns); } @Override @@ -2480,9 +2480,9 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMPool(resourcePlanName, poolPath); + rawStore.dropWMPool(resourcePlanName, poolPath, ns); } @Override @@ -2500,15 +2500,15 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + rawStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { + rawStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } public long getCacheUpdateCount() { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MWMResourcePlan.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MWMResourcePlan.java index ac51f2d77145b37da468ce8df2ac5c42f4d6c538..648587e5bf049bb59c7b4f9a53e686e961044bf3 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MWMResourcePlan.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MWMResourcePlan.java @@ -25,6 +25,7 @@ */ public class MWMResourcePlan { private String name; + private String ns; private Integer queryParallelism; private Status status; private Set pools; @@ -51,6 +52,14 @@ public String getName() { return name; } + public void setNs(String name) { + this.ns = name; + } + + public String getNs() { + return ns; + } + public void setName(String name) { this.name = name; } diff --git standalone-metastore/metastore-server/src/main/resources/package.jdo standalone-metastore/metastore-server/src/main/resources/package.jdo index fef6a42038bb2aa0cba6dfda8d710fd37cb720e7..01306e6fab8d8717e22464e501b1dd04b6c013dc 100644 --- standalone-metastore/metastore-server/src/main/resources/package.jdo +++ standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -1230,6 +1230,9 @@ + + + @@ -1241,6 +1244,7 @@ + diff --git standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql index c889bbdf96b887b29be858e41ee854f0731cd5cd..a3c4196dbff7e53be5317631b314983d16a99020 100644 --- standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -182,7 +182,7 @@ CREATE TABLE "APP"."KEY_CONSTRAINTS" ("CHILD_CD_ID" BIGINT, "CHILD_INTEGER_IDX" CREATE TABLE "APP"."METASTORE_DB_PROPERTIES" ("PROPERTY_KEY" VARCHAR(255) NOT NULL, "PROPERTY_VALUE" VARCHAR(1000) NOT NULL, "DESCRIPTION" VARCHAR(1000)); -CREATE TABLE "APP"."WM_RESOURCEPLAN" (RP_ID BIGINT NOT NULL, NAME VARCHAR(128) NOT NULL, QUERY_PARALLELISM INTEGER, STATUS VARCHAR(20) NOT NULL, DEFAULT_POOL_ID BIGINT); +CREATE TABLE "APP"."WM_RESOURCEPLAN" (RP_ID BIGINT NOT NULL, NS VARCHAR(128), NAME VARCHAR(128) NOT NULL, QUERY_PARALLELISM INTEGER, STATUS VARCHAR(20) NOT NULL, DEFAULT_POOL_ID BIGINT); CREATE TABLE "APP"."WM_POOL" (POOL_ID BIGINT NOT NULL, RP_ID BIGINT NOT NULL, PATH VARCHAR(1024) NOT NULL, ALLOC_FRACTION DOUBLE, QUERY_PARALLELISM INTEGER, SCHEDULING_POLICY VARCHAR(1024)); @@ -264,7 +264,7 @@ CREATE INDEX "APP"."CONSTRAINTS_PARENT_TBL_ID_INDEX" ON "APP"."KEY_CONSTRAINTS"( CREATE INDEX "APP"."CONSTRAINTS_CONSTRAINT_TYPE_INDEX" ON "APP"."KEY_CONSTRAINTS"("CONSTRAINT_TYPE"); -CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN" ("NAME"); +CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN" ("NS", "NAME"); CREATE UNIQUE INDEX "APP"."UNIQUE_WM_POOL" ON "APP"."WM_POOL" ("RP_ID", "PATH"); diff --git standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql index aca5227a5bb6192da6c5f070c04d2941d636bad2..bcaebd18accf86846ae44a6498046514575fc069 100644 --- standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql +++ standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql @@ -3,6 +3,13 @@ ALTER TABLE "APP"."TBLS" ADD WRITE_ID bigint DEFAULT 0; ALTER TABLE "APP"."PARTITIONS" ADD WRITE_ID bigint DEFAULT 0; +-- HIVE-20793 +ALTER TABLE "APP"."WM_RESOURCEPLAN" ADD NS VARCHAR(128); +UPDATE "APP"."WM_RESOURCEPLAN" SET NS = 'default' WHERE NS IS NULL; +DROP INDEX "APP"."UNIQUE_WM_RESOURCEPLAN"; +CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN" ("NS", "NAME"); + + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; diff --git standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 91ba134325094e413887a89e1d605efa99218288..5ea1b4450d8258e841bb4af7381ca6fb0ba1a827 100644 --- standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -642,6 +642,7 @@ CREATE TABLE WM_RESOURCEPLAN ( RP_ID bigint NOT NULL, "NAME" nvarchar(128) NOT NULL, + NS nvarchar(128), QUERY_PARALLELISM int, STATUS nvarchar(20) NOT NULL, DEFAULT_POOL_ID bigint @@ -963,7 +964,7 @@ CREATE INDEX TABLE_PARAMS_N49 ON TABLE_PARAMS (TBL_ID); -- Constraints for resource plan tables. -CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NAME"); +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NS", "NAME"); CREATE UNIQUE INDEX UNIQUE_WM_POOL ON WM_POOL (RP_ID, PATH); diff --git standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql index f0d861b3a9bc982c1e24fa49415dcfc6c105cd68..edde08db9ef7ee01800c7cc3a04c813014abdd18 100644 --- standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql +++ standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql @@ -4,6 +4,12 @@ SELECT 'Upgrading MetaStore schema from 3.2.0 to 4.0.0' AS MESSAGE; ALTER TABLE TBLS ADD WRITE_ID bigint NOT NULL DEFAULT 0; ALTER TABLE PARTITIONS ADD WRITE_ID bigint NOT NULL DEFAULT 0; +-- HIVE-20793 +ALTER TABLE WM_RESOURCEPLAN ADD NS nvarchar(128); +UPDATE WM_RESOURCEPLAN SET NS = 'default' WHERE NS IS NULL; +DROP INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN; +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NS", "NAME"); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS MESSAGE; diff --git standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 3af2ebb253f82bb85976d229d4ac2225deffdbde..5555a59c7d7e933d25d8d5af611e5b6aa0c0c19b 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -917,11 +917,12 @@ CREATE TABLE IF NOT EXISTS `METASTORE_DB_PROPERTIES` ( CREATE TABLE IF NOT EXISTS WM_RESOURCEPLAN ( `RP_ID` bigint(20) NOT NULL, `NAME` varchar(128) NOT NULL, + `NS` varchar(128), `QUERY_PARALLELISM` int(11), `STATUS` varchar(20) NOT NULL, `DEFAULT_POOL_ID` bigint(20), PRIMARY KEY (`RP_ID`), - UNIQUE KEY `UNIQUE_WM_RESOURCEPLAN` (`NAME`) + UNIQUE KEY `UNIQUE_WM_RESOURCEPLAN` (`NAME`, `NS`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE TABLE IF NOT EXISTS WM_POOL diff --git standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql index ee0f691b524a6e822ac14e09d24d3a49ae8565b1..701acb00984c61f7511dcc48053890b154575d1f 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql @@ -4,6 +4,13 @@ SELECT 'Upgrading MetaStore schema from 3.2.0 to 4.0.0' AS ' '; ALTER TABLE TBLS ADD WRITE_ID bigint DEFAULT 0; ALTER TABLE PARTITIONS ADD WRITE_ID bigint DEFAULT 0; + +-- HIVE-20793 +ALTER TABLE `WM_RESOURCEPLAN` ADD `NS` varchar(128); +UPDATE `WM_RESOURCEPLAN` SET `NS` = 'default' WHERE `NS` IS NULL; +ALTER TABLE `WM_RESOURCEPLAN` DROP KEY `UNIQUE_WM_RESOURCEPLAN`; +ALTER TABLE `WM_RESOURCEPLAN` ADD UNIQUE KEY `UNIQUE_WM_RESOURCEPLAN` (`NAME`, `NS`); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS ' '; diff --git standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index 33aa08015a9e17585c42d64d44b364be96e69eaf..b1980c5b83f16614845063516495188ebdd8c2a3 100644 --- standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -638,6 +638,7 @@ CREATE TABLE WM_RESOURCEPLAN ( RP_ID NUMBER NOT NULL, "NAME" VARCHAR2(128) NOT NULL, + NS VARCHAR2(128), QUERY_PARALLELISM NUMBER(10), STATUS VARCHAR2(20) NOT NULL, DEFAULT_POOL_ID NUMBER @@ -927,7 +928,7 @@ ALTER TABLE METASTORE_DB_PROPERTIES ADD CONSTRAINT PROPERTY_KEY_PK PRIMARY KEY ( -- Constraints for resource plan tables. -CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NAME"); +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN (NS, "NAME"); CREATE UNIQUE INDEX UNIQUE_WM_POOL ON WM_POOL (RP_ID, PATH); diff --git standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql index bbb4a39ec4f6f616c9a1a9042a35cafd45cf9796..b9f63313251ab1fa6278b862ed9e07e62b234c04 100644 --- standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql +++ standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql @@ -3,6 +3,12 @@ SELECT 'Upgrading MetaStore schema from 3.2.0 to 4.0.0' AS Status from dual; ALTER TABLE TBLS ADD WRITE_ID number DEFAULT 0 NOT NULL; ALTER TABLE PARTITIONS ADD WRITE_ID number DEFAULT 0 NOT NULL; +-- HIVE-20793 +ALTER TABLE WM_RESOURCEPLAN ADD NS VARCHAR2(128); +UPDATE WM_RESOURCEPLAN SET NS = 'default' WHERE NS IS NULL; +DROP INDEX UNIQUE_WM_RESOURCEPLAN; +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN (NS, "NAME"); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS Status from dual; diff --git standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index ea088d77fdaec85834b8fd3f01eacdfac58dd245..9040005aa82b7a8cc5c01f257ecd47a7cc97e9b2 100644 --- standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -670,6 +670,7 @@ CREATE TABLE "METASTORE_DB_PROPERTIES" CREATE TABLE "WM_RESOURCEPLAN" ( "RP_ID" bigint NOT NULL, "NAME" character varying(128) NOT NULL, + "NS" character varying(128), "QUERY_PARALLELISM" integer, "STATUS" character varying(20) NOT NULL, "DEFAULT_POOL_ID" bigint @@ -1060,7 +1061,7 @@ ALTER TABLE ONLY "WM_RESOURCEPLAN" ADD CONSTRAINT "WM_RESOURCEPLAN_pkey" PRIMARY KEY ("RP_ID"); ALTER TABLE ONLY "WM_RESOURCEPLAN" - ADD CONSTRAINT "UNIQUE_WM_RESOURCEPLAN" UNIQUE ("NAME"); + ADD CONSTRAINT "UNIQUE_WM_RESOURCEPLAN" UNIQUE ("NS", "NAME"); ALTER TABLE ONLY "WM_POOL" ADD CONSTRAINT "WM_POOL_pkey" PRIMARY KEY ("POOL_ID"); diff --git standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql index 2a2d70ae802eb6f5b9ab7f4f9519a0af30d2c5b4..0c36069d071d4b60cc338ba729da5d22e08ca8ca 100644 --- standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql +++ standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql @@ -4,6 +4,13 @@ SELECT 'Upgrading MetaStore schema from 3.2.0 to 4.0.0'; ALTER TABLE "TBLS" ADD "WRITE_ID" bigint DEFAULT 0; ALTER TABLE "PARTITIONS" ADD "WRITE_ID" bigint DEFAULT 0; + +-- HIVE-20793 +ALTER TABLE "WM_RESOURCEPLAN" ADD "NS" character varying(128); +UPDATE "WM_RESOURCEPLAN" SET "NS" = 'default' WHERE "NS" IS NULL; +ALTER TABLE "WM_RESOURCEPLAN" DROP CONSTRAINT "UNIQUE_WM_RESOURCEPLAN"; +ALTER TABLE ONLY "WM_RESOURCEPLAN" ADD CONSTRAINT "UNIQUE_WM_RESOURCEPLAN" UNIQUE ("NS", "NAME"); + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0'; diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 966979891b71f1cbfe50f56c40c35af8b304c47f..c13e538bc4500c9c8197326043ad32ab9f22dca9 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -1060,39 +1060,39 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int } @Override - public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException, + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { - return objectStore.getResourcePlan(name); + return objectStore.getResourcePlan(name, ns); } @Override - public List getAllResourcePlans() throws MetaException { - return objectStore.getAllResourcePlans(); + public List getAllResourcePlans(String ns) throws MetaException { + return objectStore.getAllResourcePlans(ns); } @Override - public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { return objectStore.alterResourcePlan( - name, resourcePlan, canActivateDisabled, canDeactivate, isReplace); + name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { - return objectStore.getActiveResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { + return objectStore.getActiveResourcePlan(ns); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { - return objectStore.validateResourcePlan(name); + return objectStore.validateResourcePlan(name, ns); } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { - objectStore.dropResourcePlan(name); + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + objectStore.dropResourcePlan(name, ns); } @Override @@ -1109,15 +1109,15 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMTrigger(resourcePlanName, triggerName); + objectStore.dropWMTrigger(resourcePlanName, triggerName, ns); } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { - return objectStore.getTriggersForResourcePlan(resourcePlanName); + return objectStore.getTriggersForResourcePlan(resourcePlanName, ns); } @Override @@ -1133,9 +1133,9 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMPool(resourcePlanName, poolPath); + objectStore.dropWMPool(resourcePlanName, poolPath, ns); } @Override @@ -1153,15 +1153,15 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + objectStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { - objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { + objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 593d562c3498660861201f58d83c27d59d184046..e943f17a365e847091454e9e37d536c712fb386f 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -1051,36 +1051,36 @@ public void createResourcePlan( } @Override - public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException { + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException { return null; } @Override - public List getAllResourcePlans() throws MetaException { + public List getAllResourcePlans(String ns) throws MetaException { return null; } @Override public WMFullResourcePlan alterResourcePlan( - String name, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, + String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws NoSuchObjectException, InvalidOperationException, MetaException { return null; } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException { + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { return null; } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name) + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { return null; } @Override - public void dropResourcePlan(String name) throws NoSuchObjectException, MetaException { + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { } @Override @@ -1093,12 +1093,12 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, MetaException { } @Override - public List getTriggersForResourcePlan(String resourcePlanName) + public List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { return null; } @@ -1114,7 +1114,7 @@ public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExists } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { } @@ -1131,13 +1131,13 @@ public void dropWMMapping(WMMapping mapping) @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { } @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { } @Override diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 4293579ad8b55d59f2230040f23e9a693d838ca7..9fe9a65677f2638c13b31cdfe468ed08979e30aa 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -51,6 +51,7 @@ import javax.security.auth.login.LoginException; import com.google.common.base.Preconditions; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -2833,34 +2834,39 @@ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName) } @Override - public WMFullResourcePlan getResourcePlan(String resourcePlanName) + public WMFullResourcePlan getResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException { WMGetResourcePlanRequest request = new WMGetResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); return client.get_resource_plan(request).getResourcePlan(); } @Override - public List getAllResourcePlans() + public List getAllResourcePlans(String ns) throws NoSuchObjectException, MetaException, TException { WMGetAllResourcePlanRequest request = new WMGetAllResourcePlanRequest(); + request.setNs(ns); return client.get_all_resource_plans(request).getResourcePlans(); } @Override - public void dropResourcePlan(String resourcePlanName) + public void dropResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException, TException { WMDropResourcePlanRequest request = new WMDropResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); client.drop_resource_plan(request); } @Override - public WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableResourcePlan resourcePlan, + public WMFullResourcePlan alterResourcePlan(String resourcePlanName, String ns, + WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) throws NoSuchObjectException, InvalidObjectException, MetaException, TException { WMAlterResourcePlanRequest request = new WMAlterResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); request.setResourcePlan(resourcePlan); request.setIsEnableAndActivate(canActivateDisabled); request.setIsForceDeactivate(isForceDeactivate); @@ -2870,15 +2876,18 @@ public WMFullResourcePlan alterResourcePlan(String resourcePlanName, WMNullableR } @Override - public WMFullResourcePlan getActiveResourcePlan() throws MetaException, TException { - return client.get_active_resource_plan(new WMGetActiveResourcePlanRequest()).getResourcePlan(); + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException, TException { + WMGetActiveResourcePlanRequest request = new WMGetActiveResourcePlanRequest(); + request.setNs(ns); + return client.get_active_resource_plan(request).getResourcePlan(); } @Override - public WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName) + public WMValidateResourcePlanResponse validateResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException, TException { WMValidateResourcePlanRequest request = new WMValidateResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); + request.setNs(ns); return client.validate_resource_plan(request); } @@ -2899,19 +2908,21 @@ public void alterWMTrigger(WMTrigger trigger) } @Override - public void dropWMTrigger(String resourcePlanName, String triggerName) + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, MetaException, TException { WMDropTriggerRequest request = new WMDropTriggerRequest(); request.setResourcePlanName(resourcePlanName); request.setTriggerName(triggerName); + request.setNs(ns); client.drop_wm_trigger(request); } @Override - public List getTriggersForResourcePlan(String resourcePlan) + public List getTriggersForResourcePlan(String resourcePlan, String ns) throws NoSuchObjectException, MetaException, TException { WMGetTriggersForResourePlanRequest request = new WMGetTriggersForResourePlanRequest(); request.setResourcePlanName(resourcePlan); + request.setNs(ns); return client.get_triggers_for_resourceplan(request).getTriggers(); } @@ -2933,11 +2944,12 @@ public void alterWMPool(WMNullablePool pool, String poolPath) } @Override - public void dropWMPool(String resourcePlanName, String poolPath) + public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, MetaException, TException { WMDropPoolRequest request = new WMDropPoolRequest(); request.setResourcePlanName(resourcePlanName); request.setPoolPath(poolPath); + request.setNs(ns); client.drop_wm_pool(request); } @@ -2960,16 +2972,18 @@ public void dropWMMapping(WMMapping mapping) @Override public void createOrDropTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath, boolean shouldDrop) throws AlreadyExistsException, NoSuchObjectException, + String poolPath, boolean shouldDrop, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException { WMCreateOrDropTriggerToPoolMappingRequest request = new WMCreateOrDropTriggerToPoolMappingRequest(); request.setResourcePlanName(resourcePlanName); request.setTriggerName(triggerName); request.setPoolPath(poolPath); request.setDrop(shouldDrop); + request.setNs(ns); client.create_or_drop_wm_trigger_to_pool_mapping(request); } + @Override public void createCatalog(Catalog catalog) throws AlreadyExistsException, InvalidObjectException, MetaException, TException {