[SOLR-10937] No space left on device when building suggester index

• Type: Bug
• Status: Open
• Priority: Major
• Resolution: Unresolved
• Affects Version/s: 6.6
• Component/s: Suggester
• Labels: filesystem suggester

Environment

Ubuntu 16
java version "1.8.0_131"
Java(TM) SE Runtime Environment (build 1.8.0_131-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.131-b11, mixed mode)

• Assignee: Unassigned
• Reporter: Matthew Caruana Galizia
• Created: 22/Jun/17 09:54
• Updated: 08/Jun/19 15:23

Description

On startup, I see the following error in my logs even though the disk has 600GB of free space available (and 3GB on the root filesystem):

2017-06-22 08:53:01.862 ERROR (searcherExecutor-8-thread-1-processing-x:kc) [   x:kc] o.a.s.h.c.SuggestComponent Exception in building suggester index for: suggester
java.io.IOException: No space left on device
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
	at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
	at sun.nio.ch.IOUtil.write(IOUtil.java:65)
	at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
	at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
	at java.nio.channels.Channels.writeFully(Channels.java:101)
	at java.nio.channels.Channels.access$000(Channels.java:61)
	at java.nio.channels.Channels$1.write(Channels.java:174)
	at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
	at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
	at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126)
	at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53)
	at org.apache.lucene.util.OfflineSorter$ByteSequencesWriter.write(OfflineSorter.java:524)
	at org.apache.lucene.util.OfflineSorter$ByteSequencesWriter.write(OfflineSorter.java:499)
	at org.apache.lucene.util.OfflineSorter$SortPartitionTask.call(OfflineSorter.java:612)
	at org.apache.lucene.util.OfflineSorter$SortPartitionTask.call(OfflineSorter.java:588)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.lucene.util.SameThreadExecutorService.execute(SameThreadExecutorService.java:34)
	at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)
	at org.apache.lucene.util.OfflineSorter.sort(OfflineSorter.java:285)
	at org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester.build(AnalyzingSuggester.java:489)
	at org.apache.lucene.search.suggest.Lookup.build(Lookup.java:190)
	at org.apache.solr.spelling.suggest.SolrSuggester.build(SolrSuggester.java:181)
	at org.apache.solr.handler.component.SuggestComponent$SuggesterListener.buildSuggesterIndex(SuggestComponent.java:524)
	at org.apache.solr.handler.component.SuggestComponent$SuggesterListener.newSearcher(SuggestComponent.java:506)
	at org.apache.solr.core.SolrCore.lambda$getSearcher$15(SolrCore.java:2249)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: java.io.IOException: No space left on device
		at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
		at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
		at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
		at sun.nio.ch.IOUtil.write(IOUtil.java:65)
		at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
		at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
		at java.nio.channels.Channels.writeFully(Channels.java:101)
		at java.nio.channels.Channels.access$000(Channels.java:61)
		at java.nio.channels.Channels$1.write(Channels.java:174)
		at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
		at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
		at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
		at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
		at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68)
		at org.apache.lucene.util.OfflineSorter$ByteSequencesWriter.close(OfflineSorter.java:532)
		at org.apache.lucene.util.OfflineSorter$SortPartitionTask.call(OfflineSorter.java:622)
		... 16 more
		Suppressed: java.io.IOException: No space left on device
			at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
			at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
			at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
			at sun.nio.ch.IOUtil.write(IOUtil.java:65)
			at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
			at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
			at java.nio.channels.Channels.writeFully(Channels.java:101)
			at java.nio.channels.Channels.access$000(Channels.java:61)
			at java.nio.channels.Channels$1.write(Channels.java:174)
			at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
			at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
			at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
			at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
			at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
			at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70)
			... 18 more
	Suppressed: java.nio.channels.ClosedChannelException
		at sun.nio.ch.FileChannelImpl.ensureOpen(FileChannelImpl.java:110)
		at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:199)
		at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
		at java.nio.channels.Channels.writeFully(Channels.java:101)
		at java.nio.channels.Channels.access$000(Channels.java:61)
		at java.nio.channels.Channels$1.write(Channels.java:174)
		at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:419)
		at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
		at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
		at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
		at java.io.FilterOutputStream.close(FilterOutputStream.java:158)
		at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70)
		at org.apache.lucene.util.OfflineSorter$SortPartitionTask.call(OfflineSorter.java:622)
		... 16 more