Uploaded image for project: 'Hadoop Common'
  1. Hadoop Common
  2. HADOOP-11985 Improve Solaris support in Hadoop
  3. HADOOP-12629

Misaligned memory accesses in CRC32 native code causes coredumps

Add voteVotersWatch issueWatchersLinkCloneUpdate Comment AuthorReplace String in CommentUpdate Comment VisibilityDelete Comments
    XMLWordPrintableJSON

    Details

    • Type: Sub-task
    • Status: Open
    • Priority: Major
    • Resolution: Unresolved
    • Affects Version/s: 3.0.0-alpha1
    • Fix Version/s: None
    • Component/s: native
    • Labels:
      None
    • Environment:

      Solaris SPARC

      Description

      Testing on Solaris SPARC produces multiple SIGBUS core dumps, which are usually due to misaligned memory access. Some representative stack traces are below:

      hadoop-hdfs-project/hadoop-hdfs/core
       --- called from signal handler with signal 10 (SIGBUS) ---
       ffffffff5d3245ec crc32c_sb8 (ffffffff, 7ada5f02f, 0, 40, ff0000, ff00) + 9c
       ffffffff5d324954 pipelined_crc32c_sb8 (ffffffff4a3fe0b4, ffffffff4a3fe0b8, ffffffff4a3fe0bc, 7ada5f02f, 200, 2) + 24
       ffffffff5d324c58 bulk_crc (7ada5f02f, 400, 7ada5f027, 2, 200, ffffffff4a3fe1b0) + 1f8
       ffffffff5d3242d0 Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChunkedSumsByteArray (1029b99e8, ffffffff4a3fe4e8, 200, 100000, ffffffff4a3fe4f8, 1f) + 1c0
       ffffffff70810dcc * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+14336
       ffffffff70810d70 * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+0
       ffffffff70806f44 * org/apache/hadoop/util/NativeCrc32.verifyChunkedSumsByteArray(II[BI[BIILjava/lang/String;J)V+15 (line 138)
       ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.verifyChunkedSums([BII[BILjava/lang/String;J)V+39 (line 691)
       ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.verifyChunkedSums(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V+59 (line 585)
       ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.verifyChunks(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)V+11 (line 914)
       ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.receivePacket()I+658 (line 1092)
       ffffffff70806cb4 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.receiveBlock(Ljava/io/DataOutputStream;Ljava/io/DataInputStream;Ljava/io/DataOutputStream;Ljava/lang/String;Lorg/apache/hadoop/hdfs/util/DataTransferThrottler;[Lorg/apache/hadoop/hdfs/protocol/DatanodeInf+97 (line 1786)
       ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/DataXceiver.writeBlock(Lorg/apache/hadoop/hdfs/protocol/ExtendedBlock;Lorg/apache/hadoop/fs/StorageType;Lorg/apache/hadoop/security/token/Token;Ljava/lang/String;[Lorg/apache/hadoop/hdfs/protocol/DatanodeInfo;[Lorg/apa+1428 (line 1403)
       ffffffff70806f44 * org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.opWriteBlock(Ljava/io/DataInputStream;)V+178 (line 327)
       ffffffff70806f44 * org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.processOp(Lorg/apache/hadoop/hdfs/protocol/datatransfer/Op;)V+72 (line 196)
       ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/DataXceiver.run()V+539 (line 778)
      
      hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/core
       --- called from signal handler with signal 10 (SIGBUS) ---
       ffffffff5a324c8c bulk_crc (7b3ef0e53, 1, 7b3ef0e4f, 0, 0, ffffffff2d5fe230) + 22c
       ffffffff5a3242d0 Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChunkedSumsByteArray (1048ec1e8, ffffffff2d5fe568, 200, 100000, ffffffff2d5fe578, 1f) + 1c0
       ffffffff70810dcc * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+14336
       ffffffff70810d70 * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+0
       ffffffff70806f44 * org/apache/hadoop/util/NativeCrc32.verifyChunkedSumsByteArray(II[BI[BIILjava/lang/String;J)V+15 (line 138)
       ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.verifyChunkedSums([BII[BILjava/lang/String;J)V+39 (line 691)
       ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.verifyChunkedSums(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V+59 (line 585)
       ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.verifyChunks(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)V+11 (line 914)
       ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.receivePacket()I+658 (line 1092)
       ffffffff70806cb4 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.receiveBlock(Ljava/io/DataOutputStream;Ljava/io/DataInputStream;Ljava/io/DataOutputStream;Ljava/lang/String;Lorg/apache/hadoop/hdfs/util/DataTransferThrottler;[Lorg/apache/hadoop/hdfs/protocol/DatanodeInf+97 (line 1786)
       ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/DataXceiver.writeBlock(Lorg/apache/hadoop/hdfs/protocol/ExtendedBlock;Lorg/apache/hadoop/fs/StorageType;Lorg/apache/hadoop/security/token/Token;Ljava/lang/String;[Lorg/apache/hadoop/hdfs/protocol/DatanodeInfo;[Lorg/apa+1428 (line 1403)
       ffffffff70806f44 * org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.opWriteBlock(Ljava/io/DataInputStream;)V+178 (line 327)
       ffffffff70806f44 * org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.processOp(Lorg/apache/hadoop/hdfs/protocol/datatransfer/Op;)V+72 (line 196)
       ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/DataXceiver.run()V+539 (line 778)
      
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/core
       --- called from signal handler with signal 10 (SIGBUS) ---
       ffffffff5ea247dc crc32_zlib_sb8 (ffffffff, 7ac9d9c39, 0, 40, ff0000, ff00) + 9c
       ffffffff5ea249ec pipelined_crc32_zlib_sb8 (ffffffff7e1fbcb4, ffffffff7e1fbcb8, ffffffff7e1fbcbc, 7ac9d9c39, 200, 3) + 24
       ffffffff5ea24b1c bulk_crc (7ac9d9c39, 1200, 7ac975448, 9, 200, 0) + bc
       ffffffff5ea24400 Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChunkedSumsByteArray (1001839e8, ffffffff7e1fc0e8, 200, 100000, ffffffff7e1fc0f8, 0) + 2f0
       ffffffff70810dcc * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+-20945
       ffffffff70810d70 * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+0
       ffffffff70806f44 * org/apache/hadoop/util/NativeCrc32.calculateChunkedSumsByteArray(II[BI[BII)V+14 (line 172)
       ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.calculateChunkedSums([BII[BI)V+35 (line 857)
       ffffffff70806f44 * org/apache/hadoop/fs/FSOutputSummer.writeChecksumChunks([BII)V+12 (line 422)
       ffffffff70806f44 * org/apache/hadoop/fs/FSOutputSummer.write1([BII)I+28 (line 245)
       ffffffff70806cb4 * org/apache/hadoop/fs/FSOutputSummer.write([BII)V+49 (line 216)
       ffffffff70806f44 * org/apache/hadoop/fs/FSDataOutputStream$PositionCache.write([BII)V+7 (line 114)
       ffffffff70806f44 * java/io/DataOutputStream.write([BII)V+7 (line 214)
       ffffffff70806f44 * org/apache/hadoop/io/file/tfile/SimpleBufferedOutputStream.write([BII)V+20 (line 112)
       ffffffff70806f44 * java/io/DataOutputStream.write([BII)V+7 (line 214)
       ffffffff70806f44 * org/apache/hadoop/io/file/tfile/Chunk$ChunkEncoder.writeChunk([BIIZ)V+24 (line 513)
       ffffffff70806f44 * org/apache/hadoop/io/file/tfile/Chunk$ChunkEncoder.close()V+18 (line 682)
       ffffffff70806f44 * java/io/FilterOutputStream.close()V+12 (line 319)
       ffffffff70806f44 * org/apache/hadoop/io/file/tfile/TFile$Writer$ValueRegister.close()V+23 (line 994)
       ffffffff70806f44 * org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat$LogWriter.append(Lorg/apache/hadoop/yarn/logaggregation/AggregatedLogFormat$LogKey;Lorg/apache/hadoop/yarn/logaggregation/AggregatedLogFormat$LogValue;)V+55 (line 892)
       ffffffff70806f44 * org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.writeSrcFileAndALog(Lorg/apache/hadoop/fs/Path;Ljava/lang/String;JLorg/apache/hadoop/fs/Path;Lorg/apache/hadoop/fs/Path;Lorg/apache/hadoop/yarn/api/records/ContainerId;)V+209 (line 320)
       ffffffff70806f44 * org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.testForCorruptedAggregatedLogs()V+114 (line 209)
      

      Full list of core files:

      hadoop-hdfs-project/hadoop-hdfs/core
      hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/core
      hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/libhdfs/core
      hadoop-hdfs-project/hadoop-hdfs-nfs/core
      hadoop-tools/hadoop-archive-logs/core
      hadoop-tools/hadoop-archives/core
      hadoop-tools/hadoop-streaming/core
      hadoop-tools/hadoop-datajoin/core
      hadoop-tools/hadoop-extras/core
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/core
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/core
      hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/core
      hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/core
      hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/core
      hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/core
      hadoop-common-project/hadoop-common/core
      

        Attachments

        Issue Links

          Activity

            People

            • Assignee:
              alanburlison Alan Burlison
              Reporter:
              alanburlison Alan Burlison

              Dates

              • Created:
                Updated:

                Issue deployment