From 3fd08a5af9ed485d82a09e0c828ef5a04ef4051b Mon Sep 17 00:00:00 2001 From: Yu-Tang Lin Date: Fri, 11 Aug 2017 01:05:58 +0800 Subject: [PATCH] YARN-6885 minor refactor the AllocationFileLoaderService.loadQueue() and AllocationFileLoaderService.reloadAllocations() --- .../fair/AllocationFileLoaderService.java | 284 ++++++++++++--------- 1 file changed, 164 insertions(+), 120 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index bc204cb..970ab0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -286,10 +286,15 @@ public synchronized void reloadAllocations() throws IOException, Node node = elements.item(i); if (node instanceof Element) { Element element = (Element)node; - if ("queue".equals(element.getTagName()) || - "pool".equals(element.getTagName())) { + String tagName = element.getTagName(); + String text = null; + Object val = null; + switch(tagName) { + case "queue": + case "pool": queueElements.add(element); - } else if ("user".equals(element.getTagName())) { + break; + case "user": String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { @@ -298,73 +303,87 @@ public synchronized void reloadAllocations() throws IOException, continue; Element field = (Element) fieldNode; if ("maxRunningApps".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - int val = Integer.parseInt(text); - userMaxApps.put(userName, val); + text = ((Text)field.getFirstChild()).getData().trim(); + val = Integer.parseInt(text); + userMaxApps.put(userName, (Integer)val); } } - } else if ("queueMaxResourcesDefault".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - Resource val = - FairSchedulerConfiguration.parseResourceConfigValue(text); - queueMaxResourcesDefault = val; - } else if ("userMaxAppsDefault".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - int val = Integer.parseInt(text); - userMaxAppsDefault = val; - } else if ("defaultFairSharePreemptionTimeout" - .equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - long val = Long.parseLong(text) * 1000L; - defaultFairSharePreemptionTimeout = val; - } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { + break; + case "queueMaxResourcesDefault": + text = ((Text)element.getFirstChild()).getData().trim(); + val = + FairSchedulerConfiguration.parseResourceConfigValue(text); + queueMaxResourcesDefault = (Resource)val; + break; + case "userMaxAppsDefault": + text = ((Text)element.getFirstChild()).getData().trim(); + val = Integer.parseInt(text); + userMaxAppsDefault =(Integer)val; + break; + case "defaultFairSharePreemptionTimeout": + text = ((Text)element.getFirstChild()).getData().trim(); + val = Long.parseLong(text) * 1000L; + defaultFairSharePreemptionTimeout = (Long)val; + break; + case "fairSharePreemptionTimeout": if (defaultFairSharePreemptionTimeout == Long.MAX_VALUE) { - String text = ((Text)element.getFirstChild()).getData().trim(); - long val = Long.parseLong(text) * 1000L; - defaultFairSharePreemptionTimeout = val; + text = ((Text)element.getFirstChild()).getData().trim(); + val = Long.parseLong(text) * 1000L; + defaultFairSharePreemptionTimeout = (Long)val; } - } else if ("defaultMinSharePreemptionTimeout" - .equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - long val = Long.parseLong(text) * 1000L; - defaultMinSharePreemptionTimeout = val; - } else if ("defaultFairSharePreemptionThreshold" - .equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - float val = Float.parseFloat(text); - val = Math.max(Math.min(val, 1.0f), 0.0f); - defaultFairSharePreemptionThreshold = val; - } else if ("queueMaxAppsDefault".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - int val = Integer.parseInt(text); - queueMaxAppsDefault = val; - } else if ("queueMaxAMShareDefault".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); - float val = Float.parseFloat(text); - val = Math.min(val, 1.0f); - queueMaxAMShareDefault = val; - } else if ("defaultQueueSchedulingPolicy".equals(element.getTagName()) - || "defaultQueueSchedulingMode".equals(element.getTagName())) { - String text = ((Text)element.getFirstChild()).getData().trim(); + break; + case "defaultMinSharePreemptionTimeout": + text = ((Text)element.getFirstChild()).getData().trim(); + val = Long.parseLong(text) * 1000L; + defaultMinSharePreemptionTimeout = (Long)val; + break; + case "defaultFairSharePreemptionThreshold": + { + text = ((Text)element.getFirstChild()).getData().trim(); + float floatVal = Float.parseFloat(text); + floatVal = Math.max(Math.min(floatVal, 1.0f), 0.0f); + defaultFairSharePreemptionThreshold = floatVal; + } + break; + case "queueMaxAppsDefault": + text = ((Text)element.getFirstChild()).getData().trim(); + val = Integer.parseInt(text); + queueMaxAppsDefault = (Integer)val; + break; + case "queueMaxAMShareDefault": + text = ((Text)element.getFirstChild()).getData().trim(); + val = Float.parseFloat(text); + val = Math.min((float)val, 1.0f); + queueMaxAMShareDefault = (float)val; + break; + case "defaultQueueSchedulingPolicy": + case "defaultQueueSchedulingMode": + text = ((Text)element.getFirstChild()).getData().trim(); if (text.equalsIgnoreCase(FifoPolicy.NAME)) { throw new AllocationConfigurationException("Bad fair scheduler " - + "config file: defaultQueueSchedulingPolicy or " - + "defaultQueueSchedulingMode can't be FIFO."); + + "config file: defaultQueueSchedulingPolicy or " + + "defaultQueueSchedulingMode can't be FIFO."); } defaultSchedPolicy = SchedulingPolicy.parse(text); - } else if ("queuePlacementPolicy".equals(element.getTagName())) { + break; + case "queuePlacementPolicy": placementPolicyElement = element; - } else if ("reservation-planner".equals(element.getTagName())) { - String text = ((Text) element.getFirstChild()).getData().trim(); + break; + case "reservation-planner": + text = ((Text) element.getFirstChild()).getData().trim(); planner = text; - } else if ("reservation-agent".equals(element.getTagName())) { - String text = ((Text) element.getFirstChild()).getData().trim(); + break; + case "reservation-agent": + text = ((Text) element.getFirstChild()).getData().trim(); reservationAgent = text; - } else if ("reservation-policy".equals(element.getTagName())) { - String text = ((Text) element.getFirstChild()).getData().trim(); + break; + case "reservation-policy": + text = ((Text) element.getFirstChild()).getData().trim(); reservationAdmissionPolicy = text; - } else { + break; + default: LOG.warn("Bad element in allocations file: " + element.getTagName()); + break; } } } @@ -493,81 +512,103 @@ private void loadQueue(String parentName, Element element, if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; - if ("minResources".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = - FairSchedulerConfiguration.parseResourceConfigValue(text); - minQueueResources.put(queueName, val); - } else if ("maxResources".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = - FairSchedulerConfiguration.parseResourceConfigValue(text); - maxQueueResources.put(queueName, val); - } else if ("maxChildResources".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - Resource val = - FairSchedulerConfiguration.parseResourceConfigValue(text); - maxChildQueueResources.put(queueName, val); - } else if ("maxRunningApps".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - int val = Integer.parseInt(text); - queueMaxApps.put(queueName, val); - } else if ("maxAMShare".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - float val = Float.parseFloat(text); - val = Math.min(val, 1.0f); - queueMaxAMShares.put(queueName, val); - } else if ("weight".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - double val = Double.parseDouble(text); - queueWeights.put(queueName, new ResourceWeights((float)val)); - } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - long val = Long.parseLong(text) * 1000L; - minSharePreemptionTimeouts.put(queueName, val); - } else if ("fairSharePreemptionTimeout".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - long val = Long.parseLong(text) * 1000L; - fairSharePreemptionTimeouts.put(queueName, val); - } else if ("fairSharePreemptionThreshold".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); - float val = Float.parseFloat(text); - val = Math.max(Math.min(val, 1.0f), 0.0f); - fairSharePreemptionThresholds.put(queueName, val); - } else if ("schedulingPolicy".equals(field.getTagName()) - || "schedulingMode".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); + String tagName = field.getTagName(); + String text = null; + Object val = null; + switch(tagName) { + case "minResources": + text = ((Text)field.getFirstChild()).getData().trim(); + val = FairSchedulerConfiguration.parseResourceConfigValue(text); + minQueueResources.put(queueName, (Resource)val); + break; + case "maxResources": + text = ((Text)field.getFirstChild()).getData().trim(); + val = FairSchedulerConfiguration.parseResourceConfigValue(text); + maxQueueResources.put(queueName, (Resource)val); + break; + case "maxChildResources": + text = ((Text)field.getFirstChild()).getData().trim(); + val = FairSchedulerConfiguration.parseResourceConfigValue(text); + maxChildQueueResources.put(queueName, (Resource)val); + break; + case "maxRunningApps": + text = ((Text)field.getFirstChild()).getData().trim(); + val = Integer.parseInt(text); + queueMaxApps.put(queueName, (Integer)val); + break; + case "maxAMShare": + text = ((Text)field.getFirstChild()).getData().trim(); + val = Float.parseFloat(text); + val = Math.min((Float)val, 1.0f); + queueMaxAMShares.put(queueName, (Float)val); + break; + case "weight": + { + text = ((Text)field.getFirstChild()).getData().trim(); + double doubleVal = Double.parseDouble(text); + queueWeights.put(queueName, new ResourceWeights((float)doubleVal)); + } + break; + case "minSharePreemptionTimeout": + text = ((Text)field.getFirstChild()).getData().trim(); + val = Long.parseLong(text) * 1000L; + minSharePreemptionTimeouts.put(queueName, (Long)val); + break; + case "fairSharePreemptionTimeout": + text = ((Text)field.getFirstChild()).getData().trim(); + val = Long.parseLong(text) * 1000L; + fairSharePreemptionTimeouts.put(queueName, (Long)val); + break; + case "fairSharePreemptionThreshold": + { + text = ((Text)field.getFirstChild()).getData().trim(); + float floatVal = Float.parseFloat(text); + floatVal = Math.max(Math.min(floatVal, 1.0f), 0.0f); + fairSharePreemptionThresholds.put(queueName, floatVal); + } + break; + case "schedulingPolicy": + case "schedulingMode": + text = ((Text)field.getFirstChild()).getData().trim(); SchedulingPolicy policy = SchedulingPolicy.parse(text); queuePolicies.put(queueName, policy); - } else if ("aclSubmitApps".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData(); + break; + case "aclSubmitApps": + text = ((Text)field.getFirstChild()).getData(); acls.put(AccessType.SUBMIT_APP, new AccessControlList(text)); - } else if ("aclAdministerApps".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData(); + break; + case "aclAdministerApps": + text = ((Text)field.getFirstChild()).getData(); acls.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(text)); - } else if ("aclAdministerReservations".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData(); + break; + case "aclAdministerReservations": + text = ((Text)field.getFirstChild()).getData(); racls.put(ReservationACL.ADMINISTER_RESERVATIONS, - new AccessControlList(text)); - } else if ("aclListReservations".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData(); - racls.put(ReservationACL.LIST_RESERVATIONS, new AccessControlList( - text)); - } else if ("aclSubmitReservations".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData(); + new AccessControlList(text)); + break; + case "aclListReservations": + text = ((Text)field.getFirstChild()).getData(); + racls.put(ReservationACL.LIST_RESERVATIONS, + new AccessControlList(text)); + break; + case "aclSubmitReservations": + text = ((Text)field.getFirstChild()).getData(); racls.put(ReservationACL.SUBMIT_RESERVATIONS, - new AccessControlList(text)); - } else if ("reservation".equals(field.getTagName())) { + new AccessControlList(text)); + break; + case "reservation": isReservable = true; reservableQueues.add(queueName); configuredQueues.get(FSQueueType.PARENT).add(queueName); - } else if ("allowPreemptionFrom".equals(field.getTagName())) { - String text = ((Text)field.getFirstChild()).getData().trim(); + break; + case "allowPreemptionFrom": + text = ((Text)field.getFirstChild()).getData().trim(); if (!Boolean.parseBoolean(text)) { nonPreemptableQueues.add(queueName); } - } else if ("queue".endsWith(field.getTagName()) || - "pool".equals(field.getTagName())) { + break; + case "queue": + case "pool": loadQueue(queueName, field, minQueueResources, maxQueueResources, maxChildQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, @@ -575,6 +616,9 @@ private void loadQueue(String parentName, Element element, queueAcls, resAcls, configuredQueues, reservableQueues, nonPreemptableQueues); isLeaf = false; + break; + default: + break; } } -- 2.10.0