Uploaded image for project: 'Kafka'
  1. Kafka
  2. KAFKA-7442

forceUnmap mmap on linux when index resize

    XMLWordPrintableJSON

    Details

    • Type: Bug
    • Status: Open
    • Priority: Major
    • Resolution: Unresolved
    • Affects Version/s: 0.10.1.1
    • Fix Version/s: None
    • Component/s: log
    • Labels:
      None

      Description

      when resize OffsetIndex or TimeIndex,We should force unmap mmap for linux platform. Rather than waiting mixedgc or fullgc to unmap MappedByteBuffer objects

      ##before full gc

      {"request":{"mbean":"java.nio:name=mapped,type=BufferPool","type":"read"},"value":{"TotalCapacity":2434496968,"MemoryUsed":2434496968,"Count":5392,"Name":"mapped","ObjectName":{"objectName":"java.nio:name=mapped,type=BufferPool"}},"timestamp":1537945759,"status":200}
      
      
      S0     S1     E      O      M     CCS    YGC     YGCT    FGC    FGCT     GCT   
      0.00 100.00  28.88   4.93  97.64  94.72     24    0.176     0    0.000    0.176
      0.00 100.00  31.37   4.93  97.64  94.72     24    0.176     0    0.000    0.176
      
      jmap -histo:live kafka_pid
      

      ###after full gc

      S0     S1     E      O      M     CCS    YGC     YGCT    FGC    FGCT     GCT   
      0.00   0.00  23.22   5.03  97.92  94.93     24    0.176     1    0.617    0.793
      0.00   0.00  25.70   5.03  97.92  94.93     24    0.176     1    0.617    0.793
      0.00   0.00  27.86   5.03  97.92  94.93     24    0.176     1    0.617    0.793
      
      {"request":{"mbean":"java.nio:name=mapped,type=BufferPool","type":"read"},"value":{"TotalCapacity":1868266036,"MemoryUsed":1868266036,"Count":5338,"Name":"mapped","ObjectName":{"objectName":"java.nio:name=mapped,type=BufferPool"}},"timestamp":1537945860,"status":200}
      
      def resize(newSize: Int) {
          inLock(lock) {
            val raf = new RandomAccessFile(_file, "rw")
            val roundedNewSize = roundDownToExactMultiple(newSize, entrySize)
            val position = mmap.position
      
            /* Windows won't let us modify the file length while the file is mmapped :-( */
            if(Os.isWindows)
              forceUnmap(mmap)
            try {
              raf.setLength(roundedNewSize)
              mmap = raf.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, roundedNewSize)
              _maxEntries = mmap.limit / entrySize
              mmap.position(position)
            } finally {
              CoreUtils.swallow(raf.close())
            }
          }
        }
      
      [2018-09-21 13:12:24,078] INFO Rolled new log segment for 'topic-265' in 2 ms. (kafka.log.Log)
      [2018-09-21 13:13:16,436] FATAL [ReplicaFetcherThread-12-15], Disk error while replicating data for topic-264 (kafka.server.ReplicaFetcherThread)
      kafka.common.KafkaStorageException: I/O exception in append to log 'topic-264'
              at kafka.log.Log.append(Log.scala:349)
              at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:130)
              at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:42)
              at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:153)
              at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:141)
              at scala.Option.foreach(Option.scala:257)
              at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:141)
              at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:138)
              at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
              at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
              at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:138)
              at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138)
              at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138)
              at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234)
              at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:136)
              at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103)
              at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63)
      Caused by: java.io.IOException: Map failed
              at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:940)
              at kafka.log.AbstractIndex.<init>(AbstractIndex.scala:61)
              at kafka.log.OffsetIndex.<init>(OffsetIndex.scala:52)
              at kafka.log.LogSegment.<init>(LogSegment.scala:67)
              at kafka.log.Log.roll(Log.scala:778)
              at kafka.log.Log.maybeRoll(Log.scala:744)
              at kafka.log.Log.append(Log.scala:405)
              ... 16 more
      Caused by: java.lang.OutOfMemoryError: Map failed
              at sun.nio.ch.FileChannelImpl.map0(Native Method)
              at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:937)
              ... 22 more
      
      

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                huxi_2b huxihx
                Reporter:
                zhaiyuyong scott.zhai
              • Votes:
                0 Vote for this issue
                Watchers:
                4 Start watching this issue

                Dates

                • Created:
                  Updated: