diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 238432e..08c6234 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -157,7 +157,6 @@ protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster protected double nodeLocalityThreshold; // Cluster threshold for node locality protected double rackLocalityThreshold; // Cluster threshold for rack locality - private FairSchedulerEventLog eventLog; // Machine-readable event log protected boolean assignMultiple; // Allocate multiple containers per // heartbeat protected int maxAssign; // Max containers to assign per heartbeat @@ -477,9 +476,6 @@ protected void setClock(Clock clock) { this.clock = clock; } - public FairSchedulerEventLog getEventLog() { - return eventLog; - } /** * Add a new application to the scheduler, with a given id, queue name, and @@ -759,7 +755,6 @@ private synchronized void nodeUpdate(RMNode nm) { if (LOG.isDebugEnabled()) { LOG.debug("nodeUpdate: " + nm + " cluster capacity: " + clusterCapacity); } - eventLog.log("HEARTBEAT", nm.getHostName()); FSSchedulerNode node = nodes.get(nm.getNodeID()); List containerInfoList = nm.pullContainerUpdates(); @@ -814,7 +809,6 @@ private synchronized void nodeUpdate(RMNode nm) { Resource assigned = sched.assignContainer(node, false); if (Resources.greaterThan(assigned, Resources.none()) || node.getReservedContainer() != null) { - eventLog.log("ASSIGN", nm.getHostName(), assigned); assignedContainers++; assignedContainer = true; break; @@ -940,8 +934,6 @@ public synchronized void reinitialize(Configuration conf, RMContext rmContext) if (!initialized) { rootMetrics = QueueMetrics.forQueue("root", null, true, conf); this.rmContext = rmContext; - this.eventLog = new FairSchedulerEventLog(); - eventLog.init(this.conf); initialized = true; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java deleted file mode 100644 index 3f6b1e8..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.log4j.DailyRollingFileAppender; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.log4j.PatternLayout; -import org.apache.log4j.spi.LoggingEvent; - -/** - * Event log used by the fair scheduler for machine-readable debug info. - * This class uses a log4j rolling file appender to write the log, but uses - * a custom tab-separated event format of the form: - *
- * DATE    EVENT_TYPE   PARAM_1   PARAM_2   ...
- * 
- * Various event types are used by the fair scheduler. The purpose of logging - * in this format is to enable tools to parse the history log easily and read - * internal scheduler variables, rather than trying to make the log human - * readable. The fair scheduler also logs human readable messages in the - * JobTracker's main log. - * - * Constructing this class creates a disabled log. It must be initialized - * using {@link FairSchedulerEventLog#init(Configuration, String)} to begin - * writing to the file. - */ -@Private -@Unstable -class FairSchedulerEventLog { - private static final Log LOG = LogFactory.getLog(FairSchedulerEventLog.class.getName()); - - /** Set to true if logging is disabled due to an error. */ - private boolean logDisabled = true; - - /** - * Log directory, set by mapred.fairscheduler.eventlog.location in conf file; - * defaults to {hadoop.log.dir}/fairscheduler. - */ - private String logDir; - - /** - * Active log file, which is {LOG_DIR}/hadoop-{user}-fairscheduler.log. - * Older files are also stored as {LOG_FILE}.date (date format YYYY-MM-DD). - */ - private String logFile; - - /** Log4j appender used to write to the log file */ - private DailyRollingFileAppender appender; - - boolean init(FairSchedulerConfiguration conf) { - try { - logDir = conf.getEventlogDir(); - File logDirFile = new File(logDir); - if (!logDirFile.exists()) { - if (!logDirFile.mkdirs()) { - throw new IOException( - "Mkdirs failed to create " + logDirFile.toString()); - } - } - String username = System.getProperty("user.name"); - logFile = String.format("%s%shadoop-%s-fairscheduler.log", - logDir, File.separator, username); - logDisabled = false; - PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n"); - appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd"); - appender.activateOptions(); - LOG.info("Initialized fair scheduler event log, logging to " + logFile); - } catch (IOException e) { - LOG.error( - "Failed to initialize fair scheduler event log. Disabling it.", e); - logDisabled = true; - } - return !(logDisabled); - } - - /** - * Log an event, writing a line in the log file of the form - *
-   * DATE    EVENT_TYPE   PARAM_1   PARAM_2   ...
-   * 
- */ - synchronized void log(String eventType, Object... params) { - try { - if (logDisabled) - return; - StringBuffer buffer = new StringBuffer(); - buffer.append(eventType); - for (Object param: params) { - buffer.append("\t"); - buffer.append(param); - } - String message = buffer.toString(); - Logger logger = Logger.getLogger(getClass()); - appender.append(new LoggingEvent("", logger, Level.INFO, message, null)); - } catch (Exception e) { - LOG.error("Failed to append to fair scheduler event log", e); - logDisabled = true; - } - } - - /** - * Flush and close the log. - */ - synchronized void shutdown() { - try { - if (appender != null) - appender.close(); - } catch (Exception e) { - LOG.error("Failed to close fair scheduler event log", e); - logDisabled = true; - } - } - - synchronized boolean isEnabled() { - return !logDisabled; - } - - public String getLogFile() { - return logFile; - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java deleted file mode 100644 index db77795..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import java.io.File; -import java.io.IOException; - -import junit.framework.Assert; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.AsyncDispatcher; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class TestFairSchedulerEventLog { - private File logFile; - private FairScheduler scheduler; - private ResourceManager resourceManager; - - @Before - public void setUp() throws IOException { - scheduler = new FairScheduler(); - - Configuration conf = new YarnConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, - ResourceScheduler.class); - conf.set("mapred.fairscheduler.eventlog.enabled", "true"); - - // All tests assume only one assignment per node update - conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); - resourceManager = new ResourceManager(); - resourceManager.init(conf); - ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); - } - - /** - * Make sure the scheduler creates the event log. - */ - @Test - public void testCreateEventLog() throws IOException { - FairSchedulerEventLog eventLog = scheduler.getEventLog(); - - logFile = new File(eventLog.getLogFile()); - Assert.assertTrue(logFile.exists()); - } - - @After - public void tearDown() { - logFile.delete(); - logFile.getParentFile().delete(); // fairscheduler/ - scheduler = null; - resourceManager = null; - } -}