diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 6c5dbd0..ad68fae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -25,7 +25,6 @@ import java.security.Principal; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; @@ -86,6 +85,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; +import org.apache.hadoop.yarn.api.records.AMBlackListingRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -93,6 +93,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeState; @@ -104,6 +105,7 @@ import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -124,6 +126,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AMBlackListingRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AMResourceRequestInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; @@ -142,6 +146,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LogAggregationContextInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo; @@ -1495,7 +1500,26 @@ protected ApplicationSubmissionContext createAppSubmissionContext( newApp.getAppNodeLabelExpression(), newApp.getAMContainerNodeLabelExpression()); appContext.setApplicationTags(newApp.getApplicationTags()); - + appContext.setAttemptFailuresValidityInterval( + newApp.getAttemptFailuresValidityInterval()); + if (newApp.getLogAggregationContextInfo() != null) { + appContext.setLogAggregationContext(createLogAggregationContext( + newApp.getLogAggregationContextInfo())); + } + String reservationIdStr = newApp.getReservationId(); + if (reservationIdStr != null && !reservationIdStr.isEmpty()) { + ReservationId reservationId = ReservationId.parseReservationId( + reservationIdStr); + appContext.setReservationID(reservationId); + } + if (newApp.getAMBlackListingRequestInfo() != null) { + appContext.setAMBlackListRequest(createAMBlackListingRequest( + newApp.getAMBlackListingRequestInfo())); + } + if (newApp.getAMResourceRequestInfo() != null) { + appContext.setAMContainerResourceRequest( + createAMContainerResourceRequest(newApp.getAMResourceRequestInfo())); + } return appContext; } @@ -1633,6 +1657,36 @@ private UserGroupInformation createKerberosUserGroupInformation( return callerUGI; } + private LogAggregationContext createLogAggregationContext( + LogAggregationContextInfo logAggregationContextInfo) { + return LogAggregationContext.newInstance( + logAggregationContextInfo.getIncludePattern(), + logAggregationContextInfo.getExcludePattern(), + logAggregationContextInfo.getRolledLogsIncludePattern(), + logAggregationContextInfo.getRolledLogsExcludePattern(), + logAggregationContextInfo.getLogAggregationPolicyClassName(), + logAggregationContextInfo.getLogAggregationPolicyParameters()); + } + + private AMBlackListingRequest createAMBlackListingRequest( + AMBlackListingRequestInfo amBlackListingRequestInfo) { + return AMBlackListingRequest.newInstance( + amBlackListingRequestInfo.getAMBlackListingEnabled(), + amBlackListingRequestInfo.getBlackListingDisableFailureThreshold()); + } + + private ResourceRequest createAMContainerResourceRequest( + AMResourceRequestInfo resourceRequestInfo) { + return ResourceRequest.newInstance( + Priority.newInstance(resourceRequestInfo.getPriority()), + resourceRequestInfo.getResourceName(), + Resource.newInstance(resourceRequestInfo.getCapability().getMemory(), + resourceRequestInfo.getCapability().getvCores()), + resourceRequestInfo.getNumContainers(), + resourceRequestInfo.getRelaxLocality(), + resourceRequestInfo.getNodeLabelExpression()); + } + @POST @Path("/delegation-token") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java new file mode 100644 index 0000000..96d9dc9 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMBlackListingRequestInfo.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class to allow users to send information required to create a + * AMBlackListingRequest which can then be used as part of the + * ApplicationSubmissionContext + * + */ +@XmlRootElement(name = "am-black-listing-requests-info") +@XmlAccessorType(XmlAccessType.FIELD) +public class AMBlackListingRequestInfo { + + @XmlElement(name = "am-black-listing-enabled") + boolean isAMBlackListingEnabled; + + @XmlElement(name = "disable-failure-threshold") + float disableFailureThreshold; + + public AMBlackListingRequestInfo() { + } + + public boolean getAMBlackListingEnabled() { + return isAMBlackListingEnabled; + } + + public void setAMBlackListingEnabled(boolean isAMBlackListingEnabled) { + this.isAMBlackListingEnabled = isAMBlackListingEnabled; + } + + public float getBlackListingDisableFailureThreshold() { + return disableFailureThreshold; + } + + public void setBlackListingDisableFailureThreshold( + float disableFailureThreshold) { + this.disableFailureThreshold = disableFailureThreshold; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMResourceRequestInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMResourceRequestInfo.java new file mode 100644 index 0000000..1969797 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AMResourceRequestInfo.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.Priority; + +/** + * Simple class to allow users to send information required to create a + * ResourceRequest which can then be used as part of the + * ApplicationSubmissionContext + * + */ +@XmlRootElement(name = "am-resource-request-info") +@XmlAccessorType(XmlAccessType.FIELD) +public class AMResourceRequestInfo { + + @XmlElement(name = "priority") + int priority; + + @XmlElement(name = "resource-name") + String hostName; + + @XmlElement(name = "resource") + ResourceInfo resource; + + @XmlElement(name = "num-of-containers") + int numContainers; + + @XmlElement(name = "relaxLocality") + boolean relaxLocality; + + @XmlElement(name = "node-label-expression") + String nodeLabelExpression; + + public AMResourceRequestInfo() { + } + + public int getPriority() { + return this.priority; + } + + public void setPriority(int priority) { + this.priority = priority; + } + + public String getResourceName() { + return this.hostName; + } + + public void setResourceName(String resourceName) { + this.hostName = resourceName; + } + + public ResourceInfo getCapability() { + return this.resource; + } + + public void setCapability(ResourceInfo capability) { + this.resource = capability; + } + + public int getNumContainers() { + return this.numContainers; + } + + public void setNumContainers(int numContainers) { + this.numContainers = numContainers; + } + + public boolean getRelaxLocality() { + return this.relaxLocality; + } + + public void setRelaxLocality(boolean relaxLocality) { + this.relaxLocality = relaxLocality; + } + + public String getNodeLabelExpression() { + return this.nodeLabelExpression; + } + + public void setNodeLabelExpression(String nodelabelExpression) { + this.nodeLabelExpression = nodelabelExpression; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java index 5278b3e..4c636f4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java @@ -78,6 +78,21 @@ @XmlElement(name = "am-container-node-label-expression") String amContainerNodeLabelExpression; + @XmlElement(name = "log-aggregation-context-info") + LogAggregationContextInfo logAggregationContextInfo; + + @XmlElement(name = "attempt-failures-validity-interval") + long attemptFailuresValidityInterval; + + @XmlElement(name = "reservation-id") + String reservationId; + + @XmlElement(name = "am-black-listing-requests-info") + AMBlackListingRequestInfo amBlackListingRequestInfo; + + @XmlElement(name = "am-resource-request-info") + AMResourceRequestInfo resourceRequestInfo; + public ApplicationSubmissionContextInfo() { applicationId = ""; applicationName = ""; @@ -91,6 +106,11 @@ public ApplicationSubmissionContextInfo() { tags = new HashSet(); appNodeLabelExpression = ""; amContainerNodeLabelExpression = ""; + logAggregationContextInfo = null; + attemptFailuresValidityInterval = -1; + reservationId = ""; + amBlackListingRequestInfo = null; + resourceRequestInfo = null; } public String getApplicationId() { @@ -149,6 +169,26 @@ public String getAMContainerNodeLabelExpression() { return amContainerNodeLabelExpression; } + public LogAggregationContextInfo getLogAggregationContextInfo() { + return logAggregationContextInfo; + } + + public long getAttemptFailuresValidityInterval() { + return attemptFailuresValidityInterval; + } + + public AMBlackListingRequestInfo getAMBlackListingRequestInfo() { + return amBlackListingRequestInfo; + } + + public String getReservationId() { + return reservationId; + } + + public AMResourceRequestInfo getAMResourceRequestInfo() { + return this.resourceRequestInfo; + } + public void setApplicationId(String applicationId) { this.applicationId = applicationId; } @@ -206,4 +246,28 @@ public void setAppNodeLabelExpression(String appNodeLabelExpression) { public void setAMContainerNodeLabelExpression(String nodeLabelExpression) { this.amContainerNodeLabelExpression = nodeLabelExpression; } + + public void setLogAggregationContextInfo( + LogAggregationContextInfo logAggregationContextInfo) { + this.logAggregationContextInfo = logAggregationContextInfo; + } + + public void setAttemptFailuresValidityInterval( + long attemptFailuresValidityInterval) { + this.attemptFailuresValidityInterval = attemptFailuresValidityInterval; + } + + public void setReservationId(String reservationId) { + this.reservationId = reservationId; + } + + public void setAMBlackListingRequestInfo( + AMBlackListingRequestInfo amBlackListingRequestInfo) { + this.amBlackListingRequestInfo = amBlackListingRequestInfo; + } + + public void setAMResourceRequestInfo( + AMResourceRequestInfo resourceRequestInfo) { + this.resourceRequestInfo = resourceRequestInfo; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java new file mode 100644 index 0000000..be0962a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LogAggregationContextInfo.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Simple class to allow users to send information required to create a + * ContainerLaunchContext which can then be used as part of the + * ApplicationSubmissionContext + * + */ +@XmlRootElement(name = "log-aggregation-context-info") +@XmlAccessorType(XmlAccessType.FIELD) +public class LogAggregationContextInfo { + + @XmlElement(name = "log-include-pattern") + String logIncludePattern; + + @XmlElement(name = "log-exclude-pattern") + String logExcludePattern; + + @XmlElement(name = "rolled-log-include-pattern") + String rolledLogsIncludePattern; + + @XmlElement(name = "rolled-log-exclude-pattern") + String rolledLogsExcludePattern; + + @XmlElement(name = "log-aggregation-policy-class-name") + String policyClassName; + + @XmlElement(name = "log-aggregation-policy-parameter") + String policyParameters; + + public LogAggregationContextInfo() { + } + + public String getIncludePattern() { + return this.logIncludePattern; + } + + public void setIncludePattern(String includePattern) { + this.logIncludePattern = includePattern; + } + + public String getExcludePattern() { + return this.logExcludePattern; + } + + public void setExcludePattern(String excludePattern) { + this.logExcludePattern = excludePattern; + } + + public String getRolledLogsIncludePattern() { + return this.rolledLogsIncludePattern; + } + + public void setRolledLogsIncludePattern( + String rolledLogsIncludePattern) { + this.rolledLogsIncludePattern = rolledLogsIncludePattern; + } + + public String getRolledLogsExcludePattern() { + return this.rolledLogsExcludePattern; + } + + public void setRolledLogsExcludePattern( + String rolledLogsExcludePattern) { + this.rolledLogsExcludePattern = rolledLogsExcludePattern; + } + + public String getLogAggregationPolicyClassName() { + return this.policyClassName; + } + + public void setLogAggregationPolicyClassName( + String className) { + this.policyClassName = className; + } + + public String getLogAggregationPolicyParameters() { + return this.policyParameters; + } + + public void setLogAggregationPolicyParameters( + String parameters) { + this.policyParameters = parameters; + } +}