Uploaded image for project: 'Apache Drill'
  1. Apache Drill
  2. DRILL-6590

DATA_WRITE ERROR: Hash Join failed to write to output file: /tmp/drill/spill/24bac407

    XMLWordPrintableJSON

    Details

    • Type: Bug
    • Status: Open
    • Priority: Major
    • Resolution: Unresolved
    • Affects Version/s: 1.14.0
    • Fix Version/s: None
    • Component/s: Execution - Flow
    • Labels:
      None

      Description

      Apache Drill 1.14.0 git.commit.id.abbrev=eb946b0

      There was enough space on /tmp, however Hash Join failed to write to spill file
      [test@qa102-45 drill-1.14.0]# clush -a df -h /tmp
      <IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
      <IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 150G 351G 30% /
      <IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
      <IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 17G 484G 4% /
      <IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
      <IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 14G 487G 3% /
      <IPADDRESS>: Filesystem Size Used Avail Use% Mounted on
      <IPADDRESS>: /dev/mapper/vg_root-lv_root 500G 13G 488G 3% /

      Stack trace from drillbit.log

      2018-07-10 18:17:51,953 [BitServer-10] WARN o.a.d.exec.rpc.control.WorkEventBus - A fragment message arrived but there was no registered listener for that message: profile {
       state: FAILED
       error {
       error_id: "6e258de2-2d4f-4b48-967d-df1b329955cd"
       endpoint {
       address: "qa102-48.qa.lab"
       user_port: 31010
       control_port: 31011
       data_port: 31012
       version: "1.14.0-SNAPSHOT"
       state: STARTUP
       }
       error_type: DATA_WRITE
       message: "DATA_WRITE ERROR: Hash Join failed to write to output file: /tmp/drill/spill/24bac407-2adb-5763-ed08-cb5714dca2c0_HashJoin_4-22-53/spill15_outer\n\nFragment 4:53\n\n[Error Id: 6e258de2-2d4f-4b48-967d-df1b329955cd on qa102-48.qa.lab:31010]"
       exception {
       exception_class: "java.nio.channels.ClosedByInterruptException"
       stack_trace {
       class_name: "..."
       line_number: 0
       method_name: "..."
       is_native_method: false
       }
       stack_trace {
       class_name: "com.google.protobuf.CodedOutputStream"
       file_name: "CodedOutputStream.java"
       line_number: 833
       method_name: "refreshBuffer"
       is_native_method: false
       }
       stack_trace {
       class_name: "com.google.protobuf.CodedOutputStream"
       file_name: "CodedOutputStream.java"
       line_number: 843
       method_name: "flush"
       is_native_method: false
       }
       stack_trace {
       class_name: "com.google.protobuf.AbstractMessageLite"
       file_name: "AbstractMessageLite.java"
       line_number: 91
       method_name: "writeDelimitedTo"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.cache.VectorSerializer$Writer"
       file_name: "VectorSerializer.java"
       line_number: 97
       method_name: "write"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
       file_name: "HashPartition.java"
       line_number: 346
       method_name: "spillThisPartition"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
       file_name: "HashPartition.java"
       line_number: 263
       method_name: "completeABatch"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
       file_name: "HashPartition.java"
       line_number: 237
       method_name: "completeAnOuterBatch"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.common.HashPartition"
       file_name: "HashPartition.java"
       line_number: 232
       method_name: "appendOuterRow"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.test.generated.HashJoinProbeGen49"
       file_name: "HashJoinProbeTemplate.java"
       line_number: 306
       method_name: "executeProbePhase"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.test.generated.HashJoinProbeGen49"
       file_name: "HashJoinProbeTemplate.java"
       line_number: 393
       method_name: "probeAndProject"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 357
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 172
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 274
       method_name: "sniffNonEmptyBatch"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 236
       method_name: "prefetchFirstBatchFromBothSides"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 216
       method_name: "buildSchema"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 152
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 274
       method_name: "sniffNonEmptyBatch"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 236
       method_name: "prefetchFirstBatchFromBothSides"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 216
       method_name: "buildSchema"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 152
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 274
       method_name: "sniffNonEmptyBatch"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 236
       method_name: "prefetchFirstBatchFromBothSides"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 216
       method_name: "buildSchema"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 152
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 274
       method_name: "sniffNonEmptyBatch"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 236
       method_name: "prefetchFirstBatchFromBothSides"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 216
       method_name: "buildSchema"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 152
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 274
       method_name: "sniffNonEmptyBatch"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 236
       method_name: "prefetchFirstBatchFromBothSides"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 216
       method_name: "buildSchema"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 152
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 109
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
       file_name: "AbstractUnaryRecordBatch.java"
       line_number: 63
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.project.ProjectRecordBatch"
       file_name: "ProjectRecordBatch.java"
       line_number: 147
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 172
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 109
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
       file_name: "AbstractUnaryRecordBatch.java"
       line_number: 63
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 172
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 109
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
       file_name: "AbstractUnaryRecordBatch.java"
       line_number: 63
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 172
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 274
       method_name: "sniffNonEmptyBatch"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 236
       method_name: "prefetchFirstBatchFromBothSides"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 216
       method_name: "buildSchema"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 152
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 274
       method_name: "sniffNonEmptyBatch"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 236
       method_name: "prefetchFirstBatchFromBothSides"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.join.HashJoinBatch"
       file_name: "HashJoinBatch.java"
       line_number: 216
       method_name: "buildSchema"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 152
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 109
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
       file_name: "AbstractUnaryRecordBatch.java"
       line_number: 63
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.project.ProjectRecordBatch"
       file_name: "ProjectRecordBatch.java"
       line_number: 147
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 172
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 109
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.aggregate.HashAggBatch"
       file_name: "HashAggBatch.java"
       line_number: 199
       method_name: "buildSchema"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 152
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 119
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 109
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractUnaryRecordBatch"
       file_name: "AbstractUnaryRecordBatch.java"
       line_number: 63
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.project.ProjectRecordBatch"
       file_name: "ProjectRecordBatch.java"
       line_number: 147
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.record.AbstractRecordBatch"
       file_name: "AbstractRecordBatch.java"
       line_number: 172
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.BaseRootExec"
       file_name: "BaseRootExec.java"
       line_number: 103
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.SingleSenderCreator$SingleSenderRootExec"
       file_name: "SingleSenderCreator.java"
       line_number: 93
       method_name: "innerNext"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.physical.impl.BaseRootExec"
       file_name: "BaseRootExec.java"
       line_number: 93
       method_name: "next"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.work.fragment.FragmentExecutor$1"
       file_name: "FragmentExecutor.java"
       line_number: 294
       method_name: "run"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.work.fragment.FragmentExecutor$1"
       file_name: "FragmentExecutor.java"
       line_number: 281
       method_name: "run"
       is_native_method: false
       }
       stack_trace {
       class_name: "..."
       line_number: 0
       method_name: "..."
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.hadoop.security.UserGroupInformation"
       file_name: "UserGroupInformation.java"
       line_number: 1595
       method_name: "doAs"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.exec.work.fragment.FragmentExecutor"
       file_name: "FragmentExecutor.java"
       line_number: 281
       method_name: "run"
       is_native_method: false
       }
       stack_trace {
       class_name: "org.apache.drill.common.SelfCleaningRunnable"
       file_name: "SelfCleaningRunnable.java"
       line_number: 38
       method_name: "run"
       is_native_method: false
       }
      

        Attachments

          Activity

            People

            • Assignee:
              Unassigned
              Reporter:
              khfaraaz Khurram Faraaz
            • Votes:
              0 Vote for this issue
              Watchers:
              2 Start watching this issue

              Dates

              • Created:
                Updated: