diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index bd0602b..a80bd22 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -95,6 +95,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SchedulerConfigurationMutationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; @@ -113,12 +114,15 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueConfigurationMutation; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SchedulerConfigurationMutation; +import org.apache.hadoop.yarn.api.records.SchedulerOperationType; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -2614,4 +2618,96 @@ public Void run() throws IOException, YarnException { app.getApplicationTimeouts().get(appTimeout.getTimeoutType())); return Response.status(Status.OK).entity(timeout).build(); } + + @PUT + @Path("/conf/scheduler/mutate") + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response mutateSchedulerConfiguration(SchedulerConfigurationMutationInfo mutationInfo, + @Context HttpServletRequest hsr) + throws AuthorizationException { + init(); + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + throw new AuthorizationException( + "Unable to obtain user name, user not authenticated"); + } + + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + return Response.status(Status.FORBIDDEN) + .entity("The default static user cannot carry out this operation.") + .build(); + } + + SchedulerConfigurationMutationRequest request = createConfigurationMutationRequest(mutationInfo); + try { + callerUGI.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, YarnException { + rm.getClientRMService().mutateSchedulerConfiguration(request); + return null; + } + }); + } catch (IOException e) { + throw new BadRequestException(e); + } + return Response.status(Status.OK).build(); + } + + private SchedulerConfigurationMutationRequest createConfigurationMutationRequest(SchedulerConfigurationMutationInfo info) { + List queueMutations = new ArrayList<>(); + for (AddQueueInfo addInfo : info.getAddQueueInfo()) { + getQueueConfigurationMutation(addInfo, queueMutations, ""); + } + for (RemoveQueueInfo removeInfo : info.getRemoveQueueInfo()) { + getQueueConfigurationMutation(removeInfo, queueMutations, ""); + } + for (UpdateQueueInfo updateInfo : info.getUpdateQueueInfo()) { + getQueueConfigurationMutation(updateInfo, queueMutations, ""); + } + SchedulerConfigurationMutation schedMutation = SchedulerConfigurationMutation.newInstance(queueMutations); + SchedulerConfigurationMutationRequest req = SchedulerConfigurationMutationRequest.newInstance(schedMutation); + return req; + } + + private void getQueueConfigurationMutation(AddQueueInfo addInfo, + List queueMutations, String parentQueue) { + String queue = parentQueue.isEmpty() ? addInfo.getQueue() : (parentQueue + "." + addInfo.getQueue()); + QueueConfigurationMutation addMutation = QueueConfigurationMutation.newInstance(queue, + SchedulerOperationType.ADD, addInfo.getParams()); + queueMutations.add(addMutation); + if (addInfo.getAddQueueInfo() != null) { + for (AddQueueInfo childAddInfo : addInfo.getAddQueueInfo()) { + getQueueConfigurationMutation(childAddInfo, queueMutations, queue); + } + } + } + + private void getQueueConfigurationMutation(RemoveQueueInfo removeInfo, + List queueMutations, String parentQueue) { + String queue = parentQueue.isEmpty() ? removeInfo.getQueue() : (parentQueue + "." + removeInfo.getQueue()); + QueueConfigurationMutation removeMutation = QueueConfigurationMutation.newInstance(queue, + SchedulerOperationType.REMOVE, Collections.emptyMap()); + queueMutations.add(removeMutation); + if (removeInfo.getRemoveQueueInfo() != null) { + for (RemoveQueueInfo childRemoveInfo : removeInfo.getRemoveQueueInfo()) { + getQueueConfigurationMutation(childRemoveInfo, queueMutations, queue); + } + } + } + + private void getQueueConfigurationMutation(UpdateQueueInfo updateInfo, + List queueMutations, String parentQueue) { + String queue = parentQueue.isEmpty() ? updateInfo.getQueueName() : (parentQueue + "." + updateInfo.getQueueName()); + QueueConfigurationMutation updateMutation = QueueConfigurationMutation.newInstance(queue, + SchedulerOperationType.UPDATE, updateInfo.getParams()); + queueMutations.add(updateMutation); + if (updateInfo.getUpdateNestedQueues() != null) { + for (UpdateQueueInfo childUpdateInfo : updateInfo.getUpdateNestedQueues()) { + getQueueConfigurationMutation(childUpdateInfo, queueMutations, queue); + } + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AddQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AddQueueInfo.java new file mode 100644 index 0000000..47576d1 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AddQueueInfo.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "add") +@XmlAccessorType(XmlAccessType.FIELD) +public class AddQueueInfo { + + @XmlElement(name = "queue") + private String queue; + + private HashMap params = new HashMap<>(); + + @XmlElement(name = "add") + private ArrayList addQueueInfo = new ArrayList<>(); + + public AddQueueInfo() { } + + public String getQueue() { + return this.queue; + } + + public HashMap getParams() { + return params; + } + + public ArrayList getAddQueueInfo() { + return addQueueInfo; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/RemoveQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/RemoveQueueInfo.java new file mode 100644 index 0000000..3726bd7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/RemoveQueueInfo.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +@XmlRootElement(name = "remove") +@XmlAccessorType(XmlAccessType.FIELD) +public class RemoveQueueInfo { + + @XmlElement(name = "queue") + private String queue; + + @XmlElement(name = "remove") + private ArrayList removeQueueInfo = new ArrayList<>(); + + public RemoveQueueInfo() { + } + + public String getQueue() { + return this.queue; + } + + public ArrayList getRemoveQueueInfo() { + return removeQueueInfo; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerConfigurationMutationInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerConfigurationMutationInfo.java new file mode 100644 index 0000000..dd8a9d1 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerConfigurationMutationInfo.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "schedConf") +@XmlAccessorType(XmlAccessType.FIELD) +public class SchedulerConfigurationMutationInfo { + + @XmlElement(name = "add") + private ArrayList addQueueInfo = new ArrayList<>(); + + @XmlElement(name = "remove") + private ArrayList removeQueueInfo = new ArrayList<>(); + + @XmlElement(name = "update") + private ArrayList updateQueueInfo = new ArrayList<>(); + + public SchedulerConfigurationMutationInfo() { + // JAXB needs this + } + + public ArrayList getAddQueueInfo() { + return addQueueInfo; + } + + public ArrayList getRemoveQueueInfo() { + return removeQueueInfo; + } + + public ArrayList getUpdateQueueInfo() { + return updateQueueInfo; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UpdateQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UpdateQueueInfo.java new file mode 100644 index 0000000..1f6368a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UpdateQueueInfo.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "update") +@XmlAccessorType(XmlAccessType.FIELD) +public class UpdateQueueInfo { + + @XmlElement(name = "name") + private String queueName; + + private HashMap params = new HashMap<>(); + + @XmlElement(name = "update") + private ArrayList updateNestedQueues = new ArrayList<>(); + + public UpdateQueueInfo() { + } + + public String getQueueName() { + return this.queueName; + } + + public HashMap getParams() { + return params; + } + + public ArrayList getUpdateNestedQueues() { + return updateNestedQueues; + } +}