[ASTERIXDB-1796] Job hanging after "no space on device"

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Component/s: CLUS - Cluster management
• Assignee: Yingyi Bu
• Reporter: Yingyi Bu
• Created: 17/Feb/17 23:57
• Updated: 24/Feb/17 06:19
• Resolved: 24/Feb/17 06:19

Description

Setting:
2 NCs
1GB RAM per NC
one partition per NC
TPC-H data: 2GB

Query:

USE tpch;

SET `compiler.joinmemory` "8MB"
SET `compiler.groupmemory` "8MB"
SET `compiler.sortmemory` "8MB"

WITH tmp AS
(
    SELECT l_orderkey, SUM(l_quantity) t_sum_quantity
    FROM  LineItem
    GROUP BY l_orderkey
)

SELECT c.c_name, c.c_custkey, o.o_orderkey, o.o_orderdate, o.o_totalprice,
       SUM(l.l_quantity) sum_quantity
FROM  LineItem l
JOIN  tmp t ON t.l_orderkey = l.l_orderkey
JOIN  Orders o ON o.o_orderkey = t.l_orderkey
JOIN  Customer c ON c.c_custkey = o.o_custkey
WHERE t.t_sum_quantity > 30
GROUP BY c.c_name, c.c_custkey, o.o_orderkey, o.o_orderdate, o.o_totalprice
ORDER BY o.o_totalprice DESC, o.o_orderdate
LIMIT 100
;

org.apache.hyracks.api.exceptions.HyracksDataException: No space left on device

at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:376)

at org.apache.hyracks.control.nc.Task.run(Task.java:308)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)

at java.lang.Thread.run(Thread.java:745)

Caused by: org.apache.hyracks.api.exceptions.HyracksDataException: No space left on device

at org.apache.hyracks.control.nc.io.IOManager.syncWrite(IOManager.java:143)

at org.apache.hyracks.dataflow.common.io.RunFileWriter.nextFrame(RunFileWriter.java:60)

at org.apache.hyracks.dataflow.std.buffermanager.VPartitionTupleBufferManager.flushPartition(VPartitionTupleBufferManager.java:285)

at org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoin.probe(OptimizedHybridHashJoin.java:514)

at org.apache.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor$ProbeAndJoinActivityNode$1.nextFrame(OptimizedHybridHashJoinOperatorDescriptor.java:405)

at org.apache.hyracks.control.nc.Task.pushFrames(Task.java:360)

... 4 more

Caused by: java.io.IOException: No space left on device

at sun.nio.ch.FileDispatcherImpl.pwrite0(Native Method)

at sun.nio.ch.FileDispatcherImpl.pwrite(FileDispatcherImpl.java:66)

at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:89)

at sun.nio.ch.IOUtil.write(IOUtil.java:65)

at sun.nio.ch.FileChannelImpl.writeInternal(FileChannelImpl.java:778)

at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:764)

at org.apache.hyracks.control.nc.io.IOManager.syncWrite(IOManager.java:130)

... 9 more