[HADOOP-12629] Misaligned memory accesses in CRC32 native code causes coredumps

• Type: Sub-task
• Status: Open
• Priority: Major
• Resolution: Unresolved
• Affects Version/s: 3.0.0-alpha1
• Component/s: native

Environment

Solaris SPARC

• Assignee: Alan Burlison
• Reporter: Alan Burlison
• Created: 09/Dec/15 14:39
• Updated: 12/May/16 18:27

Description

Testing on Solaris SPARC produces multiple SIGBUS core dumps, which are usually due to misaligned memory access. Some representative stack traces are below:

hadoop-hdfs-project/hadoop-hdfs/core
 --- called from signal handler with signal 10 (SIGBUS) ---
 ffffffff5d3245ec crc32c_sb8 (ffffffff, 7ada5f02f, 0, 40, ff0000, ff00) + 9c
 ffffffff5d324954 pipelined_crc32c_sb8 (ffffffff4a3fe0b4, ffffffff4a3fe0b8, ffffffff4a3fe0bc, 7ada5f02f, 200, 2) + 24
 ffffffff5d324c58 bulk_crc (7ada5f02f, 400, 7ada5f027, 2, 200, ffffffff4a3fe1b0) + 1f8
 ffffffff5d3242d0 Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChunkedSumsByteArray (1029b99e8, ffffffff4a3fe4e8, 200, 100000, ffffffff4a3fe4f8, 1f) + 1c0
 ffffffff70810dcc * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+14336
 ffffffff70810d70 * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+0
 ffffffff70806f44 * org/apache/hadoop/util/NativeCrc32.verifyChunkedSumsByteArray(II[BI[BIILjava/lang/String;J)V+15 (line 138)
 ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.verifyChunkedSums([BII[BILjava/lang/String;J)V+39 (line 691)
 ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.verifyChunkedSums(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V+59 (line 585)
 ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.verifyChunks(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)V+11 (line 914)
 ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.receivePacket()I+658 (line 1092)
 ffffffff70806cb4 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.receiveBlock(Ljava/io/DataOutputStream;Ljava/io/DataInputStream;Ljava/io/DataOutputStream;Ljava/lang/String;Lorg/apache/hadoop/hdfs/util/DataTransferThrottler;[Lorg/apache/hadoop/hdfs/protocol/DatanodeInf+97 (line 1786)
 ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/DataXceiver.writeBlock(Lorg/apache/hadoop/hdfs/protocol/ExtendedBlock;Lorg/apache/hadoop/fs/StorageType;Lorg/apache/hadoop/security/token/Token;Ljava/lang/String;[Lorg/apache/hadoop/hdfs/protocol/DatanodeInfo;[Lorg/apa+1428 (line 1403)
 ffffffff70806f44 * org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.opWriteBlock(Ljava/io/DataInputStream;)V+178 (line 327)
 ffffffff70806f44 * org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.processOp(Lorg/apache/hadoop/hdfs/protocol/datatransfer/Op;)V+72 (line 196)
 ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/DataXceiver.run()V+539 (line 778)

hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/core
 --- called from signal handler with signal 10 (SIGBUS) ---
 ffffffff5a324c8c bulk_crc (7b3ef0e53, 1, 7b3ef0e4f, 0, 0, ffffffff2d5fe230) + 22c
 ffffffff5a3242d0 Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChunkedSumsByteArray (1048ec1e8, ffffffff2d5fe568, 200, 100000, ffffffff2d5fe578, 1f) + 1c0
 ffffffff70810dcc * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+14336
 ffffffff70810d70 * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+0
 ffffffff70806f44 * org/apache/hadoop/util/NativeCrc32.verifyChunkedSumsByteArray(II[BI[BIILjava/lang/String;J)V+15 (line 138)
 ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.verifyChunkedSums([BII[BILjava/lang/String;J)V+39 (line 691)
 ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.verifyChunkedSums(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V+59 (line 585)
 ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.verifyChunks(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;)V+11 (line 914)
 ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.receivePacket()I+658 (line 1092)
 ffffffff70806cb4 * org/apache/hadoop/hdfs/server/datanode/BlockReceiver.receiveBlock(Ljava/io/DataOutputStream;Ljava/io/DataInputStream;Ljava/io/DataOutputStream;Ljava/lang/String;Lorg/apache/hadoop/hdfs/util/DataTransferThrottler;[Lorg/apache/hadoop/hdfs/protocol/DatanodeInf+97 (line 1786)
 ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/DataXceiver.writeBlock(Lorg/apache/hadoop/hdfs/protocol/ExtendedBlock;Lorg/apache/hadoop/fs/StorageType;Lorg/apache/hadoop/security/token/Token;Ljava/lang/String;[Lorg/apache/hadoop/hdfs/protocol/DatanodeInfo;[Lorg/apa+1428 (line 1403)
 ffffffff70806f44 * org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.opWriteBlock(Ljava/io/DataInputStream;)V+178 (line 327)
 ffffffff70806f44 * org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.processOp(Lorg/apache/hadoop/hdfs/protocol/datatransfer/Op;)V+72 (line 196)
 ffffffff70806f44 * org/apache/hadoop/hdfs/server/datanode/DataXceiver.run()V+539 (line 778)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/core
 --- called from signal handler with signal 10 (SIGBUS) ---
 ffffffff5ea247dc crc32_zlib_sb8 (ffffffff, 7ac9d9c39, 0, 40, ff0000, ff00) + 9c
 ffffffff5ea249ec pipelined_crc32_zlib_sb8 (ffffffff7e1fbcb4, ffffffff7e1fbcb8, ffffffff7e1fbcbc, 7ac9d9c39, 200, 3) + 24
 ffffffff5ea24b1c bulk_crc (7ac9d9c39, 1200, 7ac975448, 9, 200, 0) + bc
 ffffffff5ea24400 Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChunkedSumsByteArray (1001839e8, ffffffff7e1fc0e8, 200, 100000, ffffffff7e1fc0f8, 0) + 2f0
 ffffffff70810dcc * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+-20945
 ffffffff70810d70 * org/apache/hadoop/util/NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V+0
 ffffffff70806f44 * org/apache/hadoop/util/NativeCrc32.calculateChunkedSumsByteArray(II[BI[BII)V+14 (line 172)
 ffffffff70806f44 * org/apache/hadoop/util/DataChecksum.calculateChunkedSums([BII[BI)V+35 (line 857)
 ffffffff70806f44 * org/apache/hadoop/fs/FSOutputSummer.writeChecksumChunks([BII)V+12 (line 422)
 ffffffff70806f44 * org/apache/hadoop/fs/FSOutputSummer.write1([BII)I+28 (line 245)
 ffffffff70806cb4 * org/apache/hadoop/fs/FSOutputSummer.write([BII)V+49 (line 216)
 ffffffff70806f44 * org/apache/hadoop/fs/FSDataOutputStream$PositionCache.write([BII)V+7 (line 114)
 ffffffff70806f44 * java/io/DataOutputStream.write([BII)V+7 (line 214)
 ffffffff70806f44 * org/apache/hadoop/io/file/tfile/SimpleBufferedOutputStream.write([BII)V+20 (line 112)
 ffffffff70806f44 * java/io/DataOutputStream.write([BII)V+7 (line 214)
 ffffffff70806f44 * org/apache/hadoop/io/file/tfile/Chunk$ChunkEncoder.writeChunk([BIIZ)V+24 (line 513)
 ffffffff70806f44 * org/apache/hadoop/io/file/tfile/Chunk$ChunkEncoder.close()V+18 (line 682)
 ffffffff70806f44 * java/io/FilterOutputStream.close()V+12 (line 319)
 ffffffff70806f44 * org/apache/hadoop/io/file/tfile/TFile$Writer$ValueRegister.close()V+23 (line 994)
 ffffffff70806f44 * org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat$LogWriter.append(Lorg/apache/hadoop/yarn/logaggregation/AggregatedLogFormat$LogKey;Lorg/apache/hadoop/yarn/logaggregation/AggregatedLogFormat$LogValue;)V+55 (line 892)
 ffffffff70806f44 * org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.writeSrcFileAndALog(Lorg/apache/hadoop/fs/Path;Ljava/lang/String;JLorg/apache/hadoop/fs/Path;Lorg/apache/hadoop/fs/Path;Lorg/apache/hadoop/yarn/api/records/ContainerId;)V+209 (line 320)
 ffffffff70806f44 * org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.testForCorruptedAggregatedLogs()V+114 (line 209)

Full list of core files:

hadoop-hdfs-project/hadoop-hdfs/core
hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/core
hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/libhdfs/core
hadoop-hdfs-project/hadoop-hdfs-nfs/core
hadoop-tools/hadoop-archive-logs/core
hadoop-tools/hadoop-archives/core
hadoop-tools/hadoop-streaming/core
hadoop-tools/hadoop-datajoin/core
hadoop-tools/hadoop-extras/core
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/core
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/core
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/core
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/core
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/core
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/core
hadoop-common-project/hadoop-common/core

Issue Links

• relates to: HADOOP-12008 Investigate providing SPARC hardware-accelerated CRC32 code