Index: pom.xml =================================================================== --- pom.xml (revision 1463762) +++ pom.xml (working copy) @@ -57,6 +57,18 @@ sdeboy@apache.org + + + Remko Popma + remkop@yahoo.com + + log4j-async module development, testing and documentation + + +9 + + + + log4j-user Index: core/src/main/java/org/apache/logging/log4j/core/appender/rolling/RollingFileManager.java =================================================================== --- core/src/main/java/org/apache/logging/log4j/core/appender/rolling/RollingFileManager.java (revision 1463762) +++ core/src/main/java/org/apache/logging/log4j/core/appender/rolling/RollingFileManager.java (working copy) @@ -106,14 +106,18 @@ try { size = 0; initialTime = System.currentTimeMillis(); - final OutputStream os = new FileOutputStream(getFileName(), isAppend()); - setOutputStream(os); - } catch (final FileNotFoundException ex) { + createFileAfterRollover(); + } catch (final IOException ex) { LOGGER.error("FileManager (" + getFileName() + ") " + ex); } } } + protected void createFileAfterRollover() throws IOException { + final OutputStream os = new FileOutputStream(getFileName(), isAppend()); + setOutputStream(os); + } + /** * Returns the pattern processor. * @return The PatternProcessor. Index: log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastFileAppender.java =================================================================== --- log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastFileAppender.java (revision 1463762) +++ log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastFileAppender.java (working copy) @@ -80,12 +80,6 @@ // From a user's point of view, this means that all log events are // _always_ available in the log file, without incurring the overhead // of immediateFlush=true. - // - // without LOG4J2-164: - // if (event.getClass() == RingBufferLogEvent.class) { - // boolean isEndOfBatch = ((RingBufferLogEvent) event).isEndOfBatch(); - // ((FastFileManager) getManager()).setEndOfBatch(isEndOfBatch); - // } ((FastFileManager) getManager()).setEndOfBatch(event.isEndOfBatch()); super.append(event); } Index: log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastFileManager.java =================================================================== --- log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastFileManager.java (revision 1463762) +++ log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastFileManager.java (working copy) @@ -102,6 +102,16 @@ } buffer.clear(); } + + @Override + public void close() { + flush(); + try { + randomAccessFile.close(); + } catch (final IOException ex) { + LOGGER.error("Unable to close RandomAccessFile " + getName() + ". " + ex); + } + } /** * Returns the name of the File being managed. Index: log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastRollingFileAppender.java =================================================================== --- log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastRollingFileAppender.java (revision 1463762) +++ log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastRollingFileAppender.java (working copy) @@ -79,7 +79,8 @@ */ @Override public void append(final LogEvent event) { - ((RollingFileManager) getManager()).checkRollover(event); + FastRollingFileManager manager = (FastRollingFileManager) getManager(); + manager.checkRollover(event); // Leverage the nice batching behaviour of async Loggers/Appenders: // we can signal the file manager that it needs to flush the buffer @@ -87,13 +88,7 @@ // From a user's point of view, this means that all log events are // _always_ available in the log file, without incurring the overhead // of immediateFlush=true. - // - // without LOG4J2-164: - // if (event.getClass() == RingBufferLogEvent.class) { - // boolean isEndOfBatch = ((RingBufferLogEvent) event).isEndOfBatch(); - // ((FastRollingFileManager) getManager()).setEndOfBatch(isEndOfBatch); - // } - ((FastRollingFileManager) getManager()).setEndOfBatch(event.isEndOfBatch()); + manager.setEndOfBatch(event.isEndOfBatch()); super.append(event); } @@ -133,20 +128,21 @@ * @return A FastRollingFileAppender. */ @PluginFactory - public static FastRollingFileAppender createAppender(@PluginAttr("fileName") final String fileName, - @PluginAttr("filePattern") final String filePattern, - @PluginAttr("append") final String append, - @PluginAttr("name") final String name, - @PluginAttr("immediateFlush") final String immediateFlush, - @PluginElement("policy") final TriggeringPolicy policy, - @PluginElement("strategy") RolloverStrategy strategy, - @PluginElement("layout") Layout layout, - @PluginElement("filter") final Filter filter, - @PluginAttr("suppressExceptions") final String suppress, - @PluginAttr("advertise") final String advertise, - @PluginAttr("advertiseURI") final String advertiseURI, - @PluginConfiguration final Configuration config) { - + public static FastRollingFileAppender createAppender( + @PluginAttr("fileName") final String fileName, + @PluginAttr("filePattern") final String filePattern, + @PluginAttr("append") final String append, + @PluginAttr("name") final String name, + @PluginAttr("immediateFlush") final String immediateFlush, + @PluginElement("policy") final TriggeringPolicy policy, + @PluginElement("strategy") RolloverStrategy strategy, + @PluginElement("layout") Layout layout, + @PluginElement("filter") final Filter filter, + @PluginAttr("suppressExceptions") final String suppress, + @PluginAttr("advertise") final String advertise, + @PluginAttr("advertiseURI") final String advertiseURI, + @PluginConfiguration final Configuration config) { + final boolean isAppend = append == null ? true : Boolean.valueOf(append); final boolean handleExceptions = suppress == null ? true : Boolean.valueOf(suppress); final boolean isFlush = immediateFlush == null ? true : Boolean.valueOf(immediateFlush); Index: log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastRollingFileManager.java =================================================================== --- log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastRollingFileManager.java (revision 1463762) +++ log4j-async/src/main/java/org/apache/logging/log4j/async/appender/FastRollingFileManager.java (working copy) @@ -33,7 +33,7 @@ private static FastRollingFileManagerFactory factory = new FastRollingFileManagerFactory(); private final boolean isImmediateFlush; - private final RandomAccessFile randomAccessFile; + private RandomAccessFile randomAccessFile; private final ByteBuffer buffer; private ThreadLocal isEndOfBatch = new ThreadLocal(); @@ -79,6 +79,14 @@ } @Override + protected void createFileAfterRollover() throws IOException { + this.randomAccessFile = new RandomAccessFile(getFileName(), "rw"); + if (isAppend()) { + randomAccessFile.seek(randomAccessFile.length()); + } + } + + @Override public void flush() { buffer.flip(); try { @@ -89,6 +97,16 @@ } buffer.clear(); } + + @Override + public void close() { + flush(); + try { + randomAccessFile.close(); + } catch (final IOException ex) { + LOGGER.error("Unable to close RandomAccessFile " + getName() + ". " + ex); + } + } /** * Factory to create a FastRollingFileManager. Index: log4j-async/src/test/java/org/apache/logging/log4j/async/appender/FastRollingFileAppenderRolloverTest.java =================================================================== --- log4j-async/src/test/java/org/apache/logging/log4j/async/appender/FastRollingFileAppenderRolloverTest.java (revision 0) +++ log4j-async/src/test/java/org/apache/logging/log4j/async/appender/FastRollingFileAppenderRolloverTest.java (working copy) @@ -0,0 +1,75 @@ +package org.apache.logging.log4j.async.appender; + +import static org.junit.Assert.*; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.util.Arrays; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LifeCycle; +import org.apache.logging.log4j.core.config.XMLConfigurationFactory; +import org.junit.BeforeClass; +import org.junit.Test; + +public class FastRollingFileAppenderRolloverTest { + + @BeforeClass + public static void beforeClass() { + System.setProperty(XMLConfigurationFactory.CONFIGURATION_FILE_PROPERTY, + "FastRollingFileAppenderTest.xml"); + } + + @Test + public void testRollover() throws Exception { + File f = new File("FastRollingFileAppenderTest.log"); + // System.out.println(f.getAbsolutePath()); + File after1 = new File("afterRollover-1.log"); + f.delete(); + after1.delete(); + + Logger log = LogManager.getLogger("com.foo.Bar"); + String msg = "First a short message that does not trigger rollover"; + log.info(msg); + Thread.sleep(50); + + BufferedReader reader = new BufferedReader(new FileReader(f)); + String line1 = reader.readLine(); + assertTrue(line1.contains(msg)); + reader.close(); + + assertFalse("afterRollover-1.log not created yet", after1.exists()); + + String exceed = "Long message that exceeds rollover size... "; + char[] padding = new char[250]; + Arrays.fill(padding, 'X'); + exceed += new String(padding); + log.warn(exceed); + assertFalse("exceeded size but afterRollover-1.log not created yet", after1.exists()); + + String trigger = "This message triggers rollover."; + log.warn(trigger); + + ((LifeCycle) LogManager.getContext()).stop(); // stop async thread + + assertTrue("afterRollover-1.log created", after1.exists()); + + reader = new BufferedReader(new FileReader(f)); + String new1 = reader.readLine(); + assertTrue("after rollover only new msg", new1.contains(trigger)); + assertNull("No more lines", reader.readLine()); + reader.close(); + f.delete(); + + reader = new BufferedReader(new FileReader(after1)); + String old1 = reader.readLine(); + assertTrue("renamed file line 1", old1.contains(msg)); + String old2 = reader.readLine(); + assertTrue("renamed file line 2", old2.contains(exceed)); + assertNull("No more lines", reader.readLine()); + reader.close(); + after1.delete(); + } +} Index: log4j-async/src/test/java/org/apache/logging/log4j/async/perftest/PerfTestDriver.java =================================================================== --- log4j-async/src/test/java/org/apache/logging/log4j/async/perftest/PerfTestDriver.java (revision 1463762) +++ log4j-async/src/test/java/org/apache/logging/log4j/async/perftest/PerfTestDriver.java (working copy) @@ -165,16 +165,16 @@ private static Setup s(String config, String runner, String name, String... systemProperties) throws IOException { WaitStrategy wait = WaitStrategy.valueOf(System.getProperty( - "WaitStrategy", "Block")); + "WaitStrategy", "Sleep")); return new Setup(PerfTest.class, runner, name, config, 1, wait, systemProperties); } - // single-threaded performance test + // multi-threaded performance test private static Setup m(String config, String runner, String name, int threadCount, String... systemProperties) throws IOException { WaitStrategy wait = WaitStrategy.valueOf(System.getProperty( - "WaitStrategy", "Block")); + "WaitStrategy", "Sleep")); return new Setup(MTPerfTest.class, runner, name, config, threadCount, wait, systemProperties); } @@ -189,162 +189,74 @@ long start = System.nanoTime(); List tests = new ArrayList(); + // includeLocation=false tests.add(s("perf-logback.xml", LOGBK, "Sync")); tests.add(s("perf-log4j12.xml", LOG12, "Sync")); tests.add(s("perf3PlainNoLoc.xml", LOG20, "Sync")); + tests.add(s("perf3PlainNoLoc.xml", LOG20, "Loggers all async", + ALL_ASYNC, SYSCLOCK)); + tests.add(s("perf7MixedNoLoc.xml", LOG20, "Loggers mixed sync/async")); tests.add(s("perf-logback-async.xml", LOGBK, "Async Appender")); tests.add(s("perf-log4j12-async.xml", LOG12, "Async Appender")); - tests.add(s("perf5AsyncApndNoLoc.xml", LOG20, - "Async Appender no location")); - tests.add(s("perf3PlainNoLoc.xml", LOG20, "All async no loc SysClock", - ALL_ASYNC, SYSCLOCK)); - tests.add(s("perf7MixedNoLoc.xml", LOG20, "Mixed async no location")); + tests.add(s("perf5AsyncApndNoLoc.xml", LOG20, "Async Appender")); - for (int i = 32; i <= 64; i *= 2) { + // includeLocation=true + // tests.add(s("perf6AsyncApndLoc.xml", LOG20, + // "Async Appender includeLocation")); + // tests.add(s("perf8MixedLoc.xml", LOG20, + // "Mixed sync/async includeLocation")); + // tests.add(s("perf4PlainLocation.xml", LOG20, + // "Loggers all async includeLocation", ALL_ASYNC)); + // tests.add(s("perf4PlainLocation.xml", LOG20, + // "Loggers all async includeLocation CachedClock", ALL_ASYNC, + // CACHEDCLOCK)); + // tests.add(s("perf4PlainLocation.xml", LOG20, + // "Sync includeLocation")); + + // appenders + // tests.add(s("perf1syncFile.xml", LOG20, "FileAppender")); + // tests.add(s("perf1syncFastFile.xml", LOG20, "FastFileAppender")); + // tests.add(s("perf2syncRollFile.xml", LOG20, "RollFileAppender")); + // tests.add(s("perf2syncRollFastFile.xml", LOG20, + // "RollFastFileAppender")); + + final int MAX_THREADS = 16; // 64 takes a LONG time + for (int i = 2; i <= MAX_THREADS; i *= 2) { + // includeLocation = false tests.add(m("perf-logback.xml", LOGBK, "Sync", i)); tests.add(m("perf-log4j12.xml", LOG12, "Sync", i)); tests.add(m("perf3PlainNoLoc.xml", LOG20, "Sync", i)); tests.add(m("perf-logback-async.xml", LOGBK, "Async Appender", i)); tests.add(m("perf-log4j12-async.xml", LOG12, "Async Appender", i)); - tests.add(m("perf5AsyncApndNoLoc.xml", LOG20, - "Async Appender no location", i)); - tests.add(m("perf3PlainNoLoc.xml", LOG20, - "All async no loc SysClock", i, ALL_ASYNC, SYSCLOCK)); + tests.add(m("perf5AsyncApndNoLoc.xml", LOG20, "Async Appender", i)); + tests.add(m("perf3PlainNoLoc.xml", LOG20, "Loggers all async", i, + ALL_ASYNC, SYSCLOCK)); tests.add(m("perf7MixedNoLoc.xml", LOG20, - "Mixed async no location", i)); - } + "Loggers mixed sync/async", i)); - // Setup[] tests = new Setup[] { // - // s("perf-logback.xml", LOGBK, "Sync"), // - // s("perf-log4j12.xml", LOG12, "Sync"), // - // s("perf3PlainNoLoc.xml", LOG20, "Sync"), // - // s("perf-logback-async.xml", LOGBK, "Async Appender"), // - // s("perf-log4j12-async.xml", LOG12, "Async Appender"), // - // s("perf5AsyncApndNoLoc.xml", LOG20, - // "Async Appender no location"), - // s("perf3PlainNoLoc.xml", LOG20, "All async no loc SysClock", - // ALL_ASYNC, SYSCLOCK), // - // s("perf7MixedNoLoc.xml", LOG20, "Mixed async no location"), // - // - // m("perf-log4j12-async.xml", LOG12, "Async Appender", 2), // - // m("perf-log4j12.xml", LOG12, "Sync", 2), // - // m("perf-logback.xml", LOGBK, "Sync", 2), // - // m("perf-logback-async.xml", LOGBK, "Async Appender", 2), // - // m("perf3PlainNoLoc.xml", LOG20, "Sync", 2), // - // m("perf3PlainNoLoc.xml", LOG20, "All async no loc CachedClock", - // 2, ALL_ASYNC, CACHEDCLOCK), // - // m("perf3PlainNoLoc.xml", LOG20, "All async no loc SysClock", 2, - // ALL_ASYNC, SYSCLOCK), // - // - // m("perf-log4j12-async.xml", LOG12, "Async Appender", 4), // - // m("perf-log4j12.xml", LOG12, "Sync", 4), // - // m("perf-logback.xml", LOGBK, "Sync", 4), // - // m("perf-logback-async.xml", LOGBK, "Async Appender", 4), // - // m("perf3PlainNoLoc.xml", LOG20, "Sync", 4), // - // m("perf3PlainNoLoc.xml", LOG20, - // "All async no loc CachedClock", - // 4, ALL_ASYNC, CACHEDCLOCK), // - // m("perf3PlainNoLoc.xml", LOG20, "All async no loc SysClock", - // 4, - // ALL_ASYNC, SYSCLOCK), // - // - // m("perf-log4j12-async.xml", LOG12, "Async Appender", 8), // - // m("perf-log4j12.xml", LOG12, "Sync", 8), // - // m("perf-logback.xml", LOGBK, "Sync", 8), // - // m("perf-logback-async.xml", LOGBK, "Async Appender", 8), // - // m("perf3PlainNoLoc.xml", LOG20, "Sync", 8), // - // m("perf3PlainNoLoc.xml", LOG20, - // "All async no loc CachedClock", - // 8, ALL_ASYNC, CACHEDCLOCK), // - // m("perf3PlainNoLoc.xml", LOG20, "All async no loc SysClock", - // 8, - // ALL_ASYNC, SYSCLOCK), // + // includeLocation=true + // tests.add(m("perf6AsyncApndLoc.xml", LOG20, + // "Async Appender includeLocation", i)); + // tests.add(m("perf8MixedLoc.xml", LOG20, + // "Mixed sync/async includeLocation", i)); + // tests.add(m("perf4PlainLocation.xml", LOG20, + // "Loggers all async includeLocation", i, ALL_ASYNC)); + // tests.add(m("perf4PlainLocation.xml", LOG20, + // "Loggers all async includeLocation CachedClock", i, + // ALL_ASYNC, CACHEDCLOCK)); + // tests.add(m("perf4PlainLocation.xml", LOG20, + // "Sync includeLocation", i)); - // 2 threads - // m("perf5AsyncApndNoLoc.xml", LOG20,"Async Appender no location", - // 2), // - // m("perf6AsyncApndLoc.xml", - // LOG12,"Async Appender with location", - // 2), // - // m("perf7MixedNoLoc.xml", "Mixed async no location", 2), // - // m("perf8MixedLoc.xml", "Mixed async with location", 2), // - // m("perf4PlainLocation.xml", - // "All async with location SysClock", - // 2, ALL_ASYNC), // - // m("perf4PlainLocation.xml", - // "All async with location CachedClock", 2, ALL_ASYNC, - // CACHEDCLOCK), // - // m("perf4PlainLocation.xml", "All sync with location", 2), // - // m("perf1syncFile.xml", "FileAppender", 2), // - // m("perf1syncFastFile.xml", "FastFileAppender", 2), // - // m("perf2syncRollFile.xml", "RollFileAppender", 2), // - // m("perf2syncRollFastFile.xml", "RollFastFileAppender", 2), // + // appenders + // tests.add(m("perf1syncFile.xml", LOG20, "FileAppender", i)); + // tests.add(m("perf1syncFastFile.xml", LOG20, "FastFileAppender", + // i)); + // tests.add(m("perf2syncRollFile.xml", LOG20, "RollFileAppender", + // i)); + // tests.add(m("perf2syncRollFastFile.xml", LOG20, + // "RollFastFileAppender", i)); + } - // 4 threads - // m("perf5AsyncApndNoLoc.xml", LOG20,"Async Appender no location", - // 4), // - // m("perf6AsyncApndLoc.xml", "Async Appender with location", - // 4), // - // m("perf7MixedNoLoc.xml", "Mixed async no location", 4), // - // m("perf8MixedLoc.xml", "Mixed async with location", 4), // - // m("perf4PlainLocation.xml", - // "All async with location SysClock", - // 4, ALL_ASYNC), // - // m("perf4PlainLocation.xml", - // "All async with location CachedClock", 4, ALL_ASYNC, - // CACHEDCLOCK), // - // m("perf4PlainLocation.xml", "All sync with location", 4), // - // m("perf1syncFile.xml", "FileAppender", 4), // - // m("perf1syncFastFile.xml", "FastFileAppender", 4), // - // m("perf2syncRollFile.xml", "RollFileAppender", 4), // - // m("perf2syncRollFastFile.xml", "RollFastFileAppender", 4), // - - // 8 threads - // m("perf5AsyncApndNoLoc.xml", LOG20, - // "Async Appender no location", 8), // - // m("perf6AsyncApndLoc.xml", "Async Appender with location", - // 8), // - // m("perf7MixedNoLoc.xml", "Mixed async no location", 8), // - // m("perf8MixedLoc.xml", "Mixed async with location", 8), // - // m("perf4PlainLocation.xml", - // "All async with location SysClock", - // 8, ALL_ASYNC), // - // m("perf4PlainLocation.xml", - // "All async with location CachedClock", 8, ALL_ASYNC, - // CACHEDCLOCK), // - // m("perf4PlainLocation.xml", "All sync with location", 8), // - // m("perf1syncFile.xml", "FileAppender", 8), // - // m("perf1syncFastFile.xml", "FastFileAppender", 8), // - // m("perf2syncRollFile.xml", "RollFileAppender", 8), // - // m("perf2syncRollFastFile.xml", "RollFastFileAppender", 8), // - - // s("perf-log4j12-async.xml", LOG12, "Async Appender"), // - // s("perf-log4j12.xml", LOG12, "Sync"), // - // s("perf-logback.xml", LOGBK, "Sync"), // - // s("perf-logback-async.xml", LOGBK, "Async Appender"), // - // s("perf3PlainNoLoc.xml", LOG20, "Sync"), // - // s("perf3PlainNoLoc.xml", LOG20, "All async no loc CachedClock", - // ALL_ASYNC, CACHEDCLOCK), // - // s("perf3PlainNoLoc.xml", LOG20, "All async no loc SysClock", - // ALL_ASYNC, SYSCLOCK), // - // s("perf5AsyncApndNoLoc.xml", LOG20, - // "Async Appender no location"), - // - // s("perf6AsyncApndLoc.xml", "Async Appender with location"), // - // s("perf7MixedNoLoc.xml", "Mixed async no location"), // - // s("perf8MixedLoc.xml", "Mixed async with location"), // - // s("perf4PlainLocation.xml", "All async with location SysClock", - // ALL_ASYNC), // - // s("perf4PlainLocation.xml", - // "All async with location CachedClock", ALL_ASYNC, - // CACHEDCLOCK), // - // s("perf4PlainLocation.xml", "All sync with location"), // - // s("perf1syncFile.xml", "FileAppender"), // - // s("perf1syncFastFile.xml", "FastFileAppender"), // - // s("perf2syncRollFile.xml", "RollFileAppender"), // - // s("perf2syncRollFastFile.xml", "RollFastFileAppender"), // - // }; - String java = args.length > 0 ? args[0] : "java"; int repeat = args.length > 1 ? Integer.parseInt(args[1]) : 5; int x = 0; Index: log4j-async/src/test/resources/FastRollingFileAppenderTest.xml =================================================================== --- log4j-async/src/test/resources/FastRollingFileAppenderTest.xml (revision 1463762) +++ log4j-async/src/test/resources/FastRollingFileAppenderTest.xml (working copy) @@ -2,13 +2,13 @@ %d %p %c{1.} [%t] %X{aKey} %m %location %ex%n - + Index: src/site/resources/images/async-average-latency.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/site/resources/images/async-max-latency-99.99pct.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/site/resources/images/async-throughput-comparison.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/site/resources/images/async-vs-sync-throughput.png =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Index: src/site/resources/images/async-vs-sync-throughput.png =================================================================== --- src/site/resources/images/async-vs-sync-throughput.png (revision 0) +++ src/site/resources/images/async-vs-sync-throughput.png (working copy) Property changes on: src/site/resources/images/async-vs-sync-throughput.png ___________________________________________________________________ Added: svn:mime-type ## -0,0 +1 ## +application/octet-stream \ No newline at end of property Index: src/site/xdoc/performance.xml =================================================================== --- src/site/xdoc/performance.xml (revision 1463762) +++ src/site/xdoc/performance.xml (working copy) @@ -212,7 +212,38 @@ strongly encouraged to configure their applications to run in server mode when using Log4j 2.

- +
  • + Asynchronous Logging Performance Improvements +
    +

    + Log4j 2 supports a new "async" module + which offers Asynchronous Loggers for high throughput and low latency logging. + Asynchronous Loggers are implemented using the + LMAX Disruptor + inter-thread messaging library instead of the ArrayBlockingQueue used by Asynchronous Appenders. +

    + Asynchronous Appenders already offered about 5 - 10 times more throughput than + synchronous loggers, but this advantage remained more or less constant + when more threads are logging. That is, if you double the number of threads + that are logging you would expect your total throughput to increase, but + this is not the case: the throughput per thread is roughly halved so your + total throughput remains more or less the same. + (Note that this happens even if the appender queue size is large enough to hold + all messages logged during the test, so this is not caused by disk I/O.) +

    + Asynchronous Loggers have significantly higher throughput than the legacy Asynchronous Appenders, + especially in multithreaded scenarios. In one test with 64 threads, + Asynchronous Loggers were 12 times faster than the fastest Asynchronous Appender, + and 68 times faster than the fastest synchronous logger. + In addition to throughput, Asynchronous Loggers have attractive latency characteristics. + Not only is average latency lower compared to Asynchronous Appenders, + but when increasing the number of application threads that do logging, + worst-case latency remained almost constant (10 - 20 microseconds) + where Asynchronous Appenders start experiencing worst-case + latency spikes in the 100 millisecond range, a difference of four orders of magnitude. + See Asynchronous Logging Performance for details. +

    +

    • The performance results above were all derived from running the DebugDisabledPerformanceComparison, Index: src/site/xdoc/manual/async.xml =================================================================== --- src/site/xdoc/manual/async.xml (revision 1463762) +++ src/site/xdoc/manual/async.xml (working copy) @@ -479,7 +479,7 @@ ByteBuffer + RandomAccessFile instead of a BufferedOutputStream. - We saw a 10-30% performance improvement compared to + We saw a 20-200% performance improvement compared to FileAppender with "bufferedIO=true" in our measurements. Similar to the FileAppender, @@ -589,12 +589,11 @@ ByteBuffer + RandomAccessFile instead of a BufferedOutputStream. - + The FastRollingFileAppender writes to the File named in the fileName parameter @@ -843,11 +842,14 @@

    - +

    - This section shows the results of our performance tests. - The - methodology used was the same for all tests: + The performance results below were all derived from running the + PerfTest, MTPerfTest and PerfTestDriver classes which can be found + in the Log4j 2 unit test source directory. + All tests were done using the + default settings (SystemClock and SleepingWaitStrategy). + The methodology used was the same for all tests:

    • First, warm up the JVM by logging 200,000 log messages of 500 @@ -858,10 +860,42 @@ Pause for 10 microseconds * threadCount between measurements. Repeat this 5 million times, and measure average latency, latency of 99% of observations and 99.99% of observations.
      • -
    • Throughput test: measure how long it takes to log 256 * - 1024 / threadCount messages of 500 characters.
      • +
    • Throughput test: measure how long it takes to execute 256 * + 1024 / threadCount calls to Logger.log and express the + result in messages per second.
      • +
    • Repeat the test 5 times and average the results.

    Logging Throughput

    +

    + The graph below compares the throughput of synchronous loggers, + asynchronous appenders and asynchronous loggers. This + is the total throughput of all threads together. + In the test with 64 threads, asynchronous loggers are 12 times + faster than asynchronous appenders, and 68 times faster than + synchronous loggers. +

    +

    + Asynchronous loggers' throughput increases with the number of threads, + whereas both synchronous loggers and asynchronous appenders + have more or less constant throughput regardless of the number of + threads that are doing the logging. +

    +

    + +

    Asynchronous Throughput Comparison with Other Logging Packages

    +

    + We also compared throughput of asynchronous loggers to + the synchronous loggers and asynchronous appenders available + in other logging packages, specifically log4j-1.2.17 and + logback-1.0.10, with similar results. + For asynchronous appenders, total logging throughput of all + threads together remains roughly constant when adding more threads. + Asynchronous loggers make more effective use of the multiple cores + available on the machine in multi-threaded scenarios. +

    +

    + +

    On Windows 7 (64bit) with JDK1.7.0_11, 2-core Intel i5-3317u CPU @1.70Ghz with hyperthreading switched on (4 virtual cores):

    @@ -876,7 +910,7 @@ - + @@ -885,7 +919,7 @@ - + @@ -894,7 +928,7 @@ - + @@ -903,7 +937,7 @@ - + @@ -912,7 +946,7 @@ - + @@ -1046,26 +1080,13 @@
    Log4j2: Loggers all asynchronous1,655,9521,715,344 928,951 1,045,265 1,509,109
    Log4j2: Loggers mixed sync/async534,592571,099 1,204,774 1,632,204 1,368,041
    Log4j2: Async Appender1,091,5631,236,548 1,006,287 511,571 302,230
    Log4j1: Async Appender1,336,6671,373,195 911,657 636,899 406,405
    Logback: Async Appender2,231,0441,979,515 783,722 582,935 289,905
    Throughput per thread in messages/second
    -

    - In the above two environments, with the - default settings (SystemClock and SleepingWaitStrategy), - asynchronous logging - processes 6 - 68 times the number of log events in the same time - as synchronous logging. These numbers may also give you an - idea of the performance trade-off when choosing between - All Async and the more flexible - Mixed Async logging. -

    -

    Note that the numbers above are throughput per thread. - The graph below shows the total throughput of all threads together. -

    Throughput of Logging With Location (includeLocation="true")

    On Solaris 10 (64bit) with JDK1.7.0_06, 4-core Xeon X5570 dual CPU @2.93Ghz with hyperthreading switched off (8 virtual cores):

    - + @@ -1171,10 +1192,10 @@ - - - - + + + + @@ -1183,11 +1204,11 @@ -
    LoggerLogger (Log4j 2) 1 thread 2 threads 4 threads
    RollingFastFileAppender104,25439,34516,9719,465278,369213,176125,30063,103
    RollingFileAppender55,147 28,153
    Throughput per thread in operations/second + Throughput per thread in messages/second
    -

    On Solaris 10 (64bit) with JDK1.7.0_06, 4-core Xeon X5570 CPU - @2.93GHz with hyperthreading switched on (8 virtual cores):

    +

    On Solaris 10 (64bit) with JDK1.7.0_06, 4-core dual Xeon X5570 CPU + @2.93GHz with hyperthreading switched off (8 virtual cores):

    @@ -1212,10 +1233,10 @@ - - - - + + + + @@ -1224,7 +1245,7 @@ -
    Appender
    RollingFastFileAppender78,71531,42617,8818,727228,491135,35569,27732,484
    RollingFileAppender55,766 25,097
    Throughput per thread in operations/second + Throughput per thread in messages/second