Index: src/test/org/apache/lucene/index/TestByteSlices.java =================================================================== --- src/test/org/apache/lucene/index/TestByteSlices.java (revision 0) +++ src/test/org/apache/lucene/index/TestByteSlices.java (revision 0) @@ -0,0 +1,109 @@ +package org.apache.lucene.index; + +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Random; +import java.util.ArrayList; +import org.apache.lucene.util.LuceneTestCase; + +public class TestByteSlices extends LuceneTestCase { + + private static class ByteBlockAllocator extends ByteBlockPool.Allocator { + ArrayList freeByteBlocks = new ArrayList(); + + /* Allocate another byte[] from the shared pool */ + synchronized byte[] getByteBlock(boolean trackAllocations) { + final int size = freeByteBlocks.size(); + final byte[] b; + if (0 == size) + b = new byte[DocumentsWriter.BYTE_BLOCK_SIZE]; + else + b = (byte[]) freeByteBlocks.remove(size-1); + return b; + } + + /* Return a byte[] to the pool */ + synchronized void recycleByteBlocks(byte[][] blocks, int start, int end) { + for(int i=start;i 0) // Recycle all but the first buffer - docWriter.recycleByteBlocks(buffers, 1, 1+bufferUpto); + allocator.recycleByteBlocks(buffers, 1, 1+bufferUpto); // Re-use the first buffer bufferUpto = 0; @@ -82,7 +87,7 @@ System.arraycopy(buffers, 0, newBuffers, 0, buffers.length); buffers = newBuffers; } - buffer = buffers[1+bufferUpto] = docWriter.getByteBlock(trackAllocations); + buffer = buffers[1+bufferUpto] = allocator.getByteBlock(trackAllocations); bufferUpto++; byteUpto = 0; Index: src/java/org/apache/lucene/index/ByteSliceWriter.java =================================================================== --- src/java/org/apache/lucene/index/ByteSliceWriter.java (revision 0) +++ src/java/org/apache/lucene/index/ByteSliceWriter.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.lucene.index; + +final class ByteSliceWriter { + + private byte[] slice; + private int upto; + private final ByteBlockPool pool; + + int offset0; + + public ByteSliceWriter(ByteBlockPool pool) { + this.pool = pool; + } + + public void init(int address) { + slice = pool.buffers[address >> DocumentsWriter.BYTE_BLOCK_SHIFT]; + assert slice != null; + upto = address & DocumentsWriter.BYTE_BLOCK_MASK; + offset0 = address; + assert upto < slice.length; + } + + /** Write byte into byte slice stream */ + public void writeByte(byte b) { + assert slice != null; + if (slice[upto] != 0) { + upto = pool.allocSlice(slice, upto); + slice = pool.buffer; + offset0 = pool.byteOffset; + assert slice != null; + } + slice[upto++] = b; + assert upto != slice.length; + } + + public void writeBytes(final byte[] b, int offset, final int len) { + final int offsetEnd = offset + len; + while(offset < offsetEnd) { + if (slice[upto] != 0) { + // End marker + upto = pool.allocSlice(slice, upto); + slice = pool.buffer; + offset0 = pool.byteOffset; + } + + slice[upto++] = b[offset++]; + assert upto != slice.length; + } + } + + public int getAddress() { + return upto + (offset0 & DocumentsWriter.BYTE_BLOCK_NOT_MASK); + } + + public void writeVInt(int i) { + while ((i & ~0x7F) != 0) { + writeByte((byte)((i & 0x7f) | 0x80)); + i >>>= 7; + } + writeByte((byte) i); + } +} Property changes on: src/java/org/apache/lucene/index/ByteSliceWriter.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/index/DocumentsWriterFieldData.java =================================================================== --- src/java/org/apache/lucene/index/DocumentsWriterFieldData.java (revision 655033) +++ src/java/org/apache/lucene/index/DocumentsWriterFieldData.java (working copy) @@ -64,9 +64,14 @@ float boost; int postingsVectorsUpto; + final ByteSliceWriter sliceWriter; + final ByteSliceWriter vectorsSliceWriter; + public DocumentsWriterFieldData(DocumentsWriterThreadState threadState, FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; this.threadState = threadState; + sliceWriter = new ByteSliceWriter(threadState.postingsPool); + vectorsSliceWriter = new ByteSliceWriter(threadState.vectorsPool); } void resetPostingArrays() { @@ -406,15 +411,15 @@ // Now that we know doc freq for previous doc, // write it & lastDocCode - freqUpto = p.freqUpto & DocumentsWriter.BYTE_BLOCK_MASK; - freq = threadState.postingsPool.buffers[p.freqUpto >> DocumentsWriter.BYTE_BLOCK_SHIFT]; + sliceWriter.init(p.freqUpto); + if (1 == p.docFreq) - writeFreqVInt(p.lastDocCode|1); + sliceWriter.writeVInt(p.lastDocCode|1); else { - writeFreqVInt(p.lastDocCode); - writeFreqVInt(p.docFreq); + sliceWriter.writeVInt(p.lastDocCode); + sliceWriter.writeVInt(p.docFreq); } - p.freqUpto = freqUpto + (p.freqUpto & DocumentsWriter.BYTE_BLOCK_NOT_MASK); + p.freqUpto = sliceWriter.getAddress(); if (doVectors) { vector = addNewVector(); @@ -520,155 +525,37 @@ proxCode = position; } - proxUpto = p.proxUpto & DocumentsWriter.BYTE_BLOCK_MASK; - prox = threadState.postingsPool.buffers[p.proxUpto >> DocumentsWriter.BYTE_BLOCK_SHIFT]; - assert prox != null; + sliceWriter.init(p.proxUpto); if (payload != null && payload.length > 0) { - writeProxVInt((proxCode<<1)|1); - writeProxVInt(payload.length); - writeProxBytes(payload.data, payload.offset, payload.length); + sliceWriter.writeVInt((proxCode<<1)|1); + sliceWriter.writeVInt(payload.length); + sliceWriter.writeBytes(payload.data, payload.offset, payload.length); fieldInfo.storePayloads = true; } else - writeProxVInt(proxCode<<1); + sliceWriter.writeVInt(proxCode<<1); - p.proxUpto = proxUpto + (p.proxUpto & DocumentsWriter.BYTE_BLOCK_NOT_MASK); - + p.proxUpto = sliceWriter.getAddress(); p.lastPosition = position++; if (doVectorPositions) { - posUpto = vector.posUpto & DocumentsWriter.BYTE_BLOCK_MASK; - pos = threadState.vectorsPool.buffers[vector.posUpto >> DocumentsWriter.BYTE_BLOCK_SHIFT]; - writePosVInt(proxCode); - vector.posUpto = posUpto + (vector.posUpto & DocumentsWriter.BYTE_BLOCK_NOT_MASK); + vectorsSliceWriter.init(vector.posUpto); + vectorsSliceWriter.writeVInt(proxCode); + vector.posUpto = vectorsSliceWriter.getAddress(); } if (doVectorOffsets) { - offsetUpto = vector.offsetUpto & DocumentsWriter.BYTE_BLOCK_MASK; - offsets = threadState.vectorsPool.buffers[vector.offsetUpto >> DocumentsWriter.BYTE_BLOCK_SHIFT]; - writeOffsetVInt(offsetStartCode); - writeOffsetVInt(offsetEnd-offsetStart); + vectorsSliceWriter.init(vector.offsetUpto); + vectorsSliceWriter.writeVInt(offsetStartCode); + vectorsSliceWriter.writeVInt(offsetEnd-offsetStart); vector.lastOffset = offsetEnd; - vector.offsetUpto = offsetUpto + (vector.offsetUpto & DocumentsWriter.BYTE_BLOCK_NOT_MASK); + vector.offsetUpto = vectorsSliceWriter.getAddress(); } } catch (Throwable t) { throw new AbortException(t, threadState.docWriter); } } - /** Write vInt into freq stream of current Posting */ - public void writeFreqVInt(int i) { - while ((i & ~0x7F) != 0) { - writeFreqByte((byte)((i & 0x7f) | 0x80)); - i >>>= 7; - } - writeFreqByte((byte) i); - } - - /** Write vInt into prox stream of current Posting */ - public void writeProxVInt(int i) { - while ((i & ~0x7F) != 0) { - writeProxByte((byte)((i & 0x7f) | 0x80)); - i >>>= 7; - } - writeProxByte((byte) i); - } - - /** Write byte into freq stream of current Posting */ - byte[] freq; - int freqUpto; - public void writeFreqByte(byte b) { - assert freq != null; - if (freq[freqUpto] != 0) { - freqUpto = threadState.postingsPool.allocSlice(freq, freqUpto); - freq = threadState.postingsPool.buffer; - p.freqUpto = threadState.postingsPool.byteOffset; - } - freq[freqUpto++] = b; - } - - /** Write byte into prox stream of current Posting */ - byte[] prox; - int proxUpto; - public void writeProxByte(byte b) { - assert prox != null; - if (prox[proxUpto] != 0) { - proxUpto = threadState.postingsPool.allocSlice(prox, proxUpto); - prox = threadState.postingsPool.buffer; - p.proxUpto = threadState.postingsPool.byteOffset; - assert prox != null; - } - prox[proxUpto++] = b; - assert proxUpto != prox.length; - } - - /** Currently only used to copy a payload into the prox - * stream. */ - public void writeProxBytes(byte[] b, int offset, int len) { - final int offsetEnd = offset + len; - while(offset < offsetEnd) { - if (prox[proxUpto] != 0) { - // End marker - proxUpto = threadState.postingsPool.allocSlice(prox, proxUpto); - prox = threadState.postingsPool.buffer; - p.proxUpto = threadState.postingsPool.byteOffset; - } - - prox[proxUpto++] = b[offset++]; - assert proxUpto != prox.length; - } - } - - /** Write vInt into offsets stream of current - * PostingVector */ - public void writeOffsetVInt(int i) { - while ((i & ~0x7F) != 0) { - writeOffsetByte((byte)((i & 0x7f) | 0x80)); - i >>>= 7; - } - writeOffsetByte((byte) i); - } - - byte[] offsets; - int offsetUpto; - - /** Write byte into offsets stream of current - * PostingVector */ - public void writeOffsetByte(byte b) { - assert offsets != null; - if (offsets[offsetUpto] != 0) { - offsetUpto = threadState.vectorsPool.allocSlice(offsets, offsetUpto); - offsets = threadState.vectorsPool.buffer; - vector.offsetUpto = threadState.vectorsPool.byteOffset; - } - offsets[offsetUpto++] = b; - } - - /** Write vInt into pos stream of current - * PostingVector */ - public void writePosVInt(int i) { - while ((i & ~0x7F) != 0) { - writePosByte((byte)((i & 0x7f) | 0x80)); - i >>>= 7; - } - writePosByte((byte) i); - } - - byte[] pos; - int posUpto; - - /** Write byte into pos stream of current - * PostingVector */ - public void writePosByte(byte b) { - assert pos != null; - if (pos[posUpto] != 0) { - posUpto = threadState.vectorsPool.allocSlice(pos, posUpto); - pos = threadState.vectorsPool.buffer; - vector.posUpto = threadState.vectorsPool.byteOffset; - } - pos[posUpto++] = b; - } - /** Called when postings hash is too small (> 50% * occupied) or too large (< 20% occupied). */ void rehashPostings(final int newSize) { Index: src/java/org/apache/lucene/index/DocumentsWriterThreadState.java =================================================================== --- src/java/org/apache/lucene/index/DocumentsWriterThreadState.java (revision 655033) +++ src/java/org/apache/lucene/index/DocumentsWriterThreadState.java (working copy) @@ -80,8 +80,8 @@ postingsFreeList = new Posting[256]; postingsFreeCount = 0; - postingsPool = new ByteBlockPool(docWriter ,true); - vectorsPool = new ByteBlockPool(docWriter, false); + postingsPool = new ByteBlockPool(docWriter.byteBlockAllocator, true); + vectorsPool = new ByteBlockPool(docWriter.byteBlockAllocator, false); charPool = new CharBlockPool(docWriter); } Index: src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- src/java/org/apache/lucene/index/DocumentsWriter.java (revision 655033) +++ src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -1473,30 +1473,39 @@ final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1; final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK; - private ArrayList freeByteBlocks = new ArrayList(); + private class ByteBlockAllocator extends ByteBlockPool.Allocator { - /* Allocate another byte[] from the shared pool */ - synchronized byte[] getByteBlock(boolean trackAllocations) { - final int size = freeByteBlocks.size(); - final byte[] b; - if (0 == size) { - numBytesAlloc += BYTE_BLOCK_SIZE; - balanceRAM(); - b = new byte[BYTE_BLOCK_SIZE]; - } else - b = (byte[]) freeByteBlocks.remove(size-1); - if (trackAllocations) - numBytesUsed += BYTE_BLOCK_SIZE; - assert numBytesUsed <= numBytesAlloc; - return b; - } + ArrayList freeByteBlocks = new ArrayList(); + + /* Allocate another byte[] from the shared pool */ + byte[] getByteBlock(boolean trackAllocations) { + synchronized(DocumentsWriter.this) { + final int size = freeByteBlocks.size(); + final byte[] b; + if (0 == size) { + numBytesAlloc += BYTE_BLOCK_SIZE; + balanceRAM(); + b = new byte[BYTE_BLOCK_SIZE]; + } else + b = (byte[]) freeByteBlocks.remove(size-1); + if (trackAllocations) + numBytesUsed += BYTE_BLOCK_SIZE; + assert numBytesUsed <= numBytesAlloc; + return b; + } + } - /* Return byte[]'s to the pool */ - synchronized void recycleByteBlocks(byte[][] blocks, int start, int end) { - for(int i=start;i