diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java index 67a0d92..37c4bd7 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java @@ -335,11 +335,11 @@ public final class MemoryPostingsFormat extends PostingsFormat { public FSTDocsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs) { assert numDocs > 0; - if (buffer.length < bufferIn.length - bufferIn.offset) { - buffer = ArrayUtil.grow(buffer, bufferIn.length - bufferIn.offset); + if (buffer.length < bufferIn.length) { + buffer = ArrayUtil.grow(buffer, bufferIn.length); } - in.reset(buffer, 0, bufferIn.length - bufferIn.offset); - System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset); + in.reset(buffer, 0, bufferIn.length); + System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length); this.liveDocs = liveDocs; docID = -1; accum = 0; @@ -472,11 +472,11 @@ public final class MemoryPostingsFormat extends PostingsFormat { // System.out.println(" " + Integer.toHexString(bufferIn.bytes[i]&0xFF)); // } - if (buffer.length < bufferIn.length - bufferIn.offset) { - buffer = ArrayUtil.grow(buffer, bufferIn.length - bufferIn.offset); + if (buffer.length < bufferIn.length) { + buffer = ArrayUtil.grow(buffer, bufferIn.length); } in.reset(buffer, 0, bufferIn.length - bufferIn.offset); - System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset); + System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length); this.liveDocs = liveDocs; docID = -1; accum = 0; @@ -632,6 +632,7 @@ public final class MemoryPostingsFormat extends PostingsFormat { private int docFreq; private long totalTermFreq; private BytesRefFSTEnum.InputOutput current; + private BytesRef postingsSpare = new BytesRef(); public FSTTermsEnum(FieldInfo field, FST fst) { this.field = field; @@ -640,14 +641,16 @@ public final class MemoryPostingsFormat extends PostingsFormat { private void decodeMetaData() { if (!didDecode) { - buffer.reset(current.output.bytes, 0, current.output.length); + buffer.reset(current.output.bytes, current.output.offset, current.output.length); docFreq = buffer.readVInt(); if (field.getIndexOptions() != IndexOptions.DOCS_ONLY) { totalTermFreq = docFreq + buffer.readVLong(); } else { totalTermFreq = -1; } - current.output.offset = buffer.getPosition(); + postingsSpare.bytes = current.output.bytes; + postingsSpare.offset = buffer.getPosition(); + postingsSpare.length = current.output.length - (buffer.getPosition() - current.output.offset); //System.out.println(" df=" + docFreq + " totTF=" + totalTermFreq + " offset=" + buffer.getPosition() + " len=" + current.output.length); didDecode = true; } @@ -699,7 +702,7 @@ public final class MemoryPostingsFormat extends PostingsFormat { docsEnum = new FSTDocsEnum(field.getIndexOptions(), field.hasPayloads()); } } - return docsEnum.reset(current.output, liveDocs, docFreq); + return docsEnum.reset(this.postingsSpare, liveDocs, docFreq); } @Override @@ -720,7 +723,7 @@ public final class MemoryPostingsFormat extends PostingsFormat { } } //System.out.println("D&P reset this=" + this); - return docsAndPositionsEnum.reset(current.output, liveDocs, docFreq); + return docsAndPositionsEnum.reset(postingsSpare, liveDocs, docFreq); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java index 9ed1414..fbe3400 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java @@ -1707,7 +1707,6 @@ public class BlockTreeTermsReader extends FieldsProducer { if (arc.output != NO_OUTPUT) { output = fstOutputs.add(output, arc.output); } - // if (DEBUG) { // System.out.println(" index: follow label=" + toHex(target.bytes[target.offset + targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput); // } diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java index d049235..2f1df27 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java @@ -171,6 +171,8 @@ public final class FST { private final boolean allowArrayArcs; private Arc cachedRootArcs[]; + private Arc assertingCachedRootArcs[]; // only set wit assert + /** Represents a single arc. */ public final static class Arc { @@ -213,7 +215,7 @@ public final class FST { } return this; } - + boolean flag(int flag) { return FST.flag(flags, flag); } @@ -420,11 +422,18 @@ public final class FST { return node; } } - + // Caches first 128 labels @SuppressWarnings({"rawtypes","unchecked"}) private void cacheRootArcs() throws IOException { cachedRootArcs = (Arc[]) new Arc[0x80]; + readRootArcs(cachedRootArcs); + + assert setAssertingRootArcs(cachedRootArcs); + assert assertRootArcs(); + } + + public void readRootArcs(Arc[] arcs) throws IOException { final Arc arc = new Arc(); getFirstArc(arc); final BytesReader in = getBytesReader(); @@ -433,7 +442,7 @@ public final class FST { while(true) { assert arc.label != END_LABEL; if (arc.label < cachedRootArcs.length) { - cachedRootArcs[arc.label] = new Arc().copyFrom(arc); + arcs[arc.label] = new Arc().copyFrom(arc); } else { break; } @@ -444,6 +453,38 @@ public final class FST { } } } + + @SuppressWarnings({"rawtypes","unchecked"}) + private boolean setAssertingRootArcs(Arc[] arcs) throws IOException { + assertingCachedRootArcs = (Arc[]) new Arc[arcs.length]; + readRootArcs(assertingCachedRootArcs); + return true; + } + + private boolean assertRootArcs() { + assert cachedRootArcs != null; + assert assertingCachedRootArcs != null; + for (int i = 0; i < cachedRootArcs.length; i++) { + final Arc root = cachedRootArcs[i]; + final Arc asserting = assertingCachedRootArcs[i]; + if (root != null) { + assert root.arcIdx == asserting.arcIdx; + assert root.bytesPerArc == asserting.bytesPerArc; + assert root.flags == asserting.flags; + assert root.label == asserting.label; + assert root.nextArc == asserting.nextArc; + assert root.nextFinalOutput.equals(asserting.nextFinalOutput); + assert root.node == asserting.node; + assert root.numArcs == asserting.numArcs; + assert root.output.equals(asserting.output); + assert root.posArcsStart == asserting.posArcsStart; + assert root.target == asserting.target; + } else { + assert root == null && asserting == null; + } + } + return true; + } public T getEmptyOutput() { return emptyOutput; @@ -1099,7 +1140,7 @@ public final class FST { /** Finds an arc leaving the incoming arc, replacing the arc in place. * This returns null if the arc was not found, else the incoming arc. */ public Arc findTargetArc(int labelToMatch, Arc follow, Arc arc, BytesReader in) throws IOException { - assert cachedRootArcs != null; + assert assertRootArcs(); if (labelToMatch == END_LABEL) { if (follow.isFinal()) { @@ -1123,7 +1164,7 @@ public final class FST { if (follow.target == startNode && labelToMatch < cachedRootArcs.length) { final Arc result = cachedRootArcs[labelToMatch]; if (result == null) { - return result; + return null; } else { arc.copyFrom(result); return arc;