[DRILL-6590] DATA_WRITE ERROR: Hash Join failed to write to output file: /tmp/drill/spill/24bac407

• Type: Bug
• Status: Open
• Priority: Major
• Resolution: Unresolved
• Affects Version/s: 1.14.0
• Component/s: Execution - Flow
• Assignee: Unassigned
• Reporter: Khurram Faraaz
• Created: 11/Jul/18 07:33
• Updated: 12/Jul/18 07:04

Description

Apache Drill 1.14.0 git.commit.id.abbrev=eb946b0

There was enough space on /tmp, however Hash Join failed to write to spill file
[test@qa102-45 drill-1.14.0]# clush -a df -h /tmp
<IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
<IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 150G 351G 30% /
<IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
<IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 17G 484G 4% /
<IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
<IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 14G 487G 3% /
<IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
<IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 13G 488G 3% /

Stack trace from drillbit.log

2018-07-10 18:17:51,953 [BitServer-10] WARN o.a.d.exec.rpc.control.WorkEventBus - A fragment message arrived but there was no registered listener for that message: profile {
 state: FAILED
 error {
 error_id: "6e258de2-2d4f-4b48-967d-df1b329955cd"
 endpoint {
 address: "qa102-48.qa.lab"
 user_port: 31010
 control_port: 31011
 data_port: 31012
 version: "1.14.0-SNAPSHOT"
 state: STARTUP
 }
 error_type: DATA_WRITE
 message: "DATA_WRITE ERROR: Hash Join failed to write to output file: /tmp/drill/spill/24bac407-2adb-5763-ed08-cb5714dca2c0_HashJoin_4-22-53/spill15_outer\n\nFragment 4:53\n\n[Error Id: 6e258de2-2d4f-4b48-967d-df1b329955cd on qa102-48.qa.lab:31010]"
 exception {
 exception_class: "java.nio.channels.ClosedByInterruptException"
 stack_trace {
 class_name: "..."
 line_number: 0
 method_name: "..."
 is_native_method: false
 }
 stack_trace {
 class_name: "com.google.protobuf.CodedOutputStream"
 file_name: "CodedOutputStream.java"
 line_number: 833
 method_name: "refreshBuffer"
 is_native_method: false
 }
 stack_trace {
 class_name: "com.google.protobuf.CodedOutputStream"
 file_name: "CodedOutputStream.java"
 line_number: 843
 method_name: "flush"
 is_native_method: false
 }
 stack_trace {
 class_name: "com.google.protobuf.AbstractMessageLite"
 file_name: "AbstractMessageLite.java"
 line_number: 91
 method_name: "writeDelimitedTo"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.cache.VectorSerializer$Writer"
 file_name: "VectorSerializer.java"
 line_number: 97
 method_name: "write"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
 file_name: "HashPartition.java"
 line_number: 346
 method_name: "spillThisPartition"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
 file_name: "HashPartition.java"
 line_number: 263
 method_name: "completeABatch"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
 file_name: "HashPartition.java"
 line_number: 237
 method_name: "completeAnOuterBatch"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
 file_name: "HashPartition.java"
 line_number: 232
 method_name: "appendOuterRow"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.test.generated.HashJoinProbeGen49"
 file_name: "HashJoinProbeTemplate.java"
 line_number: 306
 method_name: "executeProbePhase"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.test.generated.HashJoinProbeGen49"
 file_name: "HashJoinProbeTemplate.java"
 line_number: 393
 method_name: "probeAndProject"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 357
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 172
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 274
 method_name: "sniffNonEmptyBatch"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 236
 method_name: "prefetchFirstBatchFromBothSides"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 216
 method_name: "buildSchema"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 152
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 274
 method_name: "sniffNonEmptyBatch"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 236
 method_name: "prefetchFirstBatchFromBothSides"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 216
 method_name: "buildSchema"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 152
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 274
 method_name: "sniffNonEmptyBatch"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 236
 method_name: "prefetchFirstBatchFromBothSides"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 216
 method_name: "buildSchema"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 152
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 274
 method_name: "sniffNonEmptyBatch"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 236
 method_name: "prefetchFirstBatchFromBothSides"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 216
 method_name: "buildSchema"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 152
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 274
 method_name: "sniffNonEmptyBatch"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 236
 method_name: "prefetchFirstBatchFromBothSides"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 216
 method_name: "buildSchema"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 152
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 109
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
 file_name: "AbstractUnaryRecordBatch.java"
 line_number: 63
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.project.ProjectRecordBatch"
 file_name: "ProjectRecordBatch.java"
 line_number: 147
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 172
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 109
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
 file_name: "AbstractUnaryRecordBatch.java"
 line_number: 63
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 172
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 109
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
 file_name: "AbstractUnaryRecordBatch.java"
 line_number: 63
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 172
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 274
 method_name: "sniffNonEmptyBatch"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 236
 method_name: "prefetchFirstBatchFromBothSides"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 216
 method_name: "buildSchema"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 152
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 274
 method_name: "sniffNonEmptyBatch"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 236
 method_name: "prefetchFirstBatchFromBothSides"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
 file_name: "HashJoinBatch.java"
 line_number: 216
 method_name: "buildSchema"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 152
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 109
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
 file_name: "AbstractUnaryRecordBatch.java"
 line_number: 63
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.project.ProjectRecordBatch"
 file_name: "ProjectRecordBatch.java"
 line_number: 147
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 172
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 109
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.aggregate.HashAggBatch"
 file_name: "HashAggBatch.java"
 line_number: 199
 method_name: "buildSchema"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 152
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 119
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 109
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
 file_name: "AbstractUnaryRecordBatch.java"
 line_number: 63
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.project.ProjectRecordBatch"
 file_name: "ProjectRecordBatch.java"
 line_number: 147
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
 file_name: "AbstractRecordBatch.java"
 line_number: 172
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.BaseRootExec"
 file_name: "BaseRootExec.java"
 line_number: 103
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.SingleSenderCreator$SingleSenderRootExec"
 file_name: "SingleSenderCreator.java"
 line_number: 93
 method_name: "innerNext"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.physical.impl.BaseRootExec"
 file_name: "BaseRootExec.java"
 line_number: 93
 method_name: "next"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.work.fragment.FragmentExecutor$1"
 file_name: "FragmentExecutor.java"
 line_number: 294
 method_name: "run"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.work.fragment.FragmentExecutor$1"
 file_name: "FragmentExecutor.java"
 line_number: 281
 method_name: "run"
 is_native_method: false
 }
 stack_trace {
 class_name: "..."
 line_number: 0
 method_name: "..."
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.hadoop.security.UserGroupInformation"
 file_name: "UserGroupInformation.java"
 line_number: 1595
 method_name: "doAs"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.exec.work.fragment.FragmentExecutor"
 file_name: "FragmentExecutor.java"
 line_number: 281
 method_name: "run"
 is_native_method: false
 }
 stack_trace {
 class_name: "org.apache.drill.common.SelfCleaningRunnable"
 file_name: "SelfCleaningRunnable.java"
 line_number: 38
 method_name: "run"
 is_native_method: false
 }