Index: lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (revision 1516384) +++ lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (working copy) @@ -483,7 +483,7 @@ } /** Wraps a SortedSetDocValues but with additional asserts */ - public static class AssertingSortedSetDocValues extends SortedSetDocValues { + public static final class AssertingSortedSetDocValues extends SortedSetDocValues { private final SortedSetDocValues in; private final int maxDoc; private final long valueCount; @@ -536,6 +536,11 @@ assert key.isValid(); return result; } + + @Override + public AssertingSortedSetDocValues clone() { + return new AssertingSortedSetDocValues(in.clone(), maxDoc); + } } @Override Index: lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (revision 1516384) +++ lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (working copy) @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map.Entry; import java.util.Map; @@ -2948,6 +2949,60 @@ dir.close(); } + public void testSortedSetCloning() throws IOException { + assumeTrue("Codec does not support SORTED_SET", defaultCodecSupportsSortedSet()); + Directory dir = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf); + final int numValues = _TestUtil.nextInt(random(), 1, 100); + final Document doc = new Document(); + final Set values = new HashSet(); + while(values.size() < numValues) { + values.add(_TestUtil.randomSimpleString(random())); + } + for (String value : values) { + doc.add(new SortedSetDocValuesField("field", new BytesRef(value))); + } + writer.addDocument(doc); + final AtomicReader reader = getOnlySegmentReader(writer.getReader()); + final SortedSetDocValues ssetdv1 = reader.getSortedSetDocValues("field"); + final SortedSetDocValues ssetdv2 = reader.getSortedSetDocValues("field"); + assertNotSame(ssetdv1, ssetdv2); + ssetdv1.setDocument(0); + ssetdv2.setDocument(0); + for (int i = 0, j = 0, k = 0; k < 10000; ++k) { + if (rarely()) { + if (random().nextBoolean()) { + ssetdv1.setDocument(0); + i = 0; + } else { + ssetdv2.setDocument(0); + j = 0; + } + } + if (random().nextBoolean()) { + if (i == numValues) { + assertEquals(SortedSetDocValues.NO_MORE_ORDS, ssetdv1.nextOrd()); + ssetdv1.setDocument(0); + i = 0; + } else { + assertEquals(i++, ssetdv1.nextOrd()); + } + } else { + if (j == numValues) { + assertEquals(SortedSetDocValues.NO_MORE_ORDS, ssetdv2.nextOrd()); + ssetdv2.setDocument(0); + j = 0; + } else { + assertEquals(j++, ssetdv2.nextOrd()); + } + } + } + writer.close(); + reader.close(); + dir.close(); + } + protected boolean codecAcceptsHugeBinaryValues(String field) { return true; } Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (revision 1516384) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (working copy) @@ -278,7 +278,7 @@ } } - private static class SortingSortedSetDocValues extends SortedSetDocValues { + private static final class SortingSortedSetDocValues extends SortedSetDocValues { private final SortedSetDocValues in; private final Sorter.DocMap docMap; @@ -312,6 +312,11 @@ public long lookupTerm(BytesRef key) { return in.lookupTerm(key); } + + @Override + public SortedSetDocValues clone() { + return new SortingSortedSetDocValues(in.clone(), docMap); + } } static class SortingDocsEnum extends FilterDocsEnum { Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (revision 1516384) +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (working copy) @@ -358,76 +358,96 @@ // valid: assert field != null; - final IndexInput in = data.clone(); - final BytesRef scratch = new BytesRef(); - final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT)); - - return new SortedSetDocValues() { - String[] currentOrds = new String[0]; - int currentIndex = 0; - - @Override - public long nextOrd() { - if (currentIndex == currentOrds.length) { - return NO_MORE_ORDS; + return new SimpleTextSortedSetDocValues(data.clone(), field, maxDoc); + } + + private static class SimpleTextSortedSetDocValues extends SortedSetDocValues { + + final IndexInput in; + final OneField field; + final int maxDoc; + final BytesRef scratch; + final DecimalFormat decoder; + + String[] currentOrds = new String[0]; + int currentIndex = 0; + + SimpleTextSortedSetDocValues(IndexInput in, OneField field, int maxDoc) { + this.in = in; + this.field = field; + this.maxDoc = maxDoc; + decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT)); + scratch = new BytesRef(); + } + + @Override + public long nextOrd() { + if (currentIndex == currentOrds.length) { + return NO_MORE_ORDS; + } else { + return Long.parseLong(currentOrds[currentIndex++]); + } + } + + @Override + public void setDocument(int docID) { + if (docID < 0 || docID >= maxDoc) { + throw new IndexOutOfBoundsException("docID must be 0 .. " + (maxDoc-1) + "; got " + docID); + } + try { + in.seek(field.dataStartFilePointer + field.numValues * (9 + field.pattern.length() + field.maxLength) + docID * (1 + field.ordPattern.length())); + SimpleTextUtil.readLine(in, scratch); + String ordList = scratch.utf8ToString().trim(); + if (ordList.isEmpty()) { + currentOrds = new String[0]; } else { - return Long.parseLong(currentOrds[currentIndex++]); + currentOrds = ordList.split(","); } + currentIndex = 0; + } catch (IOException ioe) { + throw new RuntimeException(ioe); } + } - @Override - public void setDocument(int docID) { - if (docID < 0 || docID >= maxDoc) { - throw new IndexOutOfBoundsException("docID must be 0 .. " + (maxDoc-1) + "; got " + docID); + @Override + public void lookupOrd(long ord, BytesRef result) { + try { + if (ord < 0 || ord >= field.numValues) { + throw new IndexOutOfBoundsException("ord must be 0 .. " + (field.numValues-1) + "; got " + ord); } + in.seek(field.dataStartFilePointer + ord * (9 + field.pattern.length() + field.maxLength)); + SimpleTextUtil.readLine(in, scratch); + assert StringHelper.startsWith(scratch, LENGTH): "got " + scratch.utf8ToString() + " in=" + in; + int len; try { - in.seek(field.dataStartFilePointer + field.numValues * (9 + field.pattern.length() + field.maxLength) + docID * (1 + field.ordPattern.length())); - SimpleTextUtil.readLine(in, scratch); - String ordList = scratch.utf8ToString().trim(); - if (ordList.isEmpty()) { - currentOrds = new String[0]; - } else { - currentOrds = ordList.split(","); - } - currentIndex = 0; - } catch (IOException ioe) { - throw new RuntimeException(ioe); + len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, "UTF-8")).intValue(); + } catch (ParseException pe) { + CorruptIndexException e = new CorruptIndexException("failed to parse int length (resource=" + in + ")"); + e.initCause(pe); + throw e; } + result.bytes = new byte[len]; + result.offset = 0; + result.length = len; + in.readBytes(result.bytes, 0, len); + } catch (IOException ioe) { + throw new RuntimeException(ioe); } + } - @Override - public void lookupOrd(long ord, BytesRef result) { - try { - if (ord < 0 || ord >= field.numValues) { - throw new IndexOutOfBoundsException("ord must be 0 .. " + (field.numValues-1) + "; got " + ord); - } - in.seek(field.dataStartFilePointer + ord * (9 + field.pattern.length() + field.maxLength)); - SimpleTextUtil.readLine(in, scratch); - assert StringHelper.startsWith(scratch, LENGTH): "got " + scratch.utf8ToString() + " in=" + in; - int len; - try { - len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, "UTF-8")).intValue(); - } catch (ParseException pe) { - CorruptIndexException e = new CorruptIndexException("failed to parse int length (resource=" + in + ")"); - e.initCause(pe); - throw e; - } - result.bytes = new byte[len]; - result.offset = 0; - result.length = len; - in.readBytes(result.bytes, 0, len); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } + @Override + public long getValueCount() { + return field.numValues; + } - @Override - public long getValueCount() { - return field.numValues; - } - }; + @Override + public SortedSetDocValues clone() { + SimpleTextSortedSetDocValues clone = (SimpleTextSortedSetDocValues) super.clone(); + clone.currentOrds = new String[0]; + return clone; + } } - + @Override public Bits getDocsWithField(FieldInfo field) throws IOException { switch (field.getDocValuesType()) { Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (revision 1516384) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (working copy) @@ -376,89 +376,127 @@ if (entry.numOrds == 0) { return SortedSetDocValues.EMPTY; // empty FST! } - FST instance; + FST fst; synchronized(this) { - instance = fstInstances.get(field.number); - if (instance == null) { + fst = fstInstances.get(field.number); + if (fst == null) { data.seek(entry.offset); - instance = new FST(data, PositiveIntOutputs.getSingleton()); - fstInstances.put(field.number, instance); + fst = new FST(data, PositiveIntOutputs.getSingleton()); + fstInstances.put(field.number, fst); } } final BinaryDocValues docToOrds = getBinary(field); - final FST fst = instance; - - // per-thread resources - final BytesReader in = fst.getBytesReader(); - final Arc firstArc = new Arc(); - final Arc scratchArc = new Arc(); - final IntsRef scratchInts = new IntsRef(); - final BytesRefFSTEnum fstEnum = new BytesRefFSTEnum(fst); - final BytesRef ref = new BytesRef(); - final ByteArrayDataInput input = new ByteArrayDataInput(); - return new SortedSetDocValues() { - long currentOrd; - @Override - public long nextOrd() { - if (input.eof()) { - return NO_MORE_ORDS; - } else { - currentOrd += input.readVLong(); - return currentOrd; - } + return new MemorySortedSetDocValues(docToOrds, fst, entry.numOrds); + } + + private static final class MemorySortedSetDocValues extends SortedSetDocValues { + + final BinaryDocValues docToOrds; + final FST fst; + final long numOrds; + + final BytesReader in; + final Arc firstArc; + final Arc scratchArc; + final IntsRef scratchInts; + final BytesRefFSTEnum fstEnum; + final BytesRef ref; + final ByteArrayDataInput input; + + long currentOrd; + + MemorySortedSetDocValues(BinaryDocValues docToOrds, FST fst, long numOrds) { + this.docToOrds = docToOrds; + this.fst = fst; + this.numOrds = numOrds; + in = fst.getBytesReader(); + firstArc = new Arc(); + scratchArc = new Arc(); + scratchInts = new IntsRef(); + fstEnum = new BytesRefFSTEnum(fst); + ref = new BytesRef(); + input = new ByteArrayDataInput(); + } + + MemorySortedSetDocValues(MemorySortedSetDocValues other) { + docToOrds = other.docToOrds; + fst = other.fst; + numOrds = other.numOrds; + in = other.in; + firstArc = other.firstArc; + scratchArc = other.scratchArc; + scratchInts = other.scratchInts; + fstEnum = other.fstEnum; + ref = other.ref; + input = new ByteArrayDataInput(); + } + + @Override + public long nextOrd() { + if (input.eof()) { + return NO_MORE_ORDS; + } else { + currentOrd += input.readVLong(); + return currentOrd; } - - @Override - public void setDocument(int docID) { - docToOrds.get(docID, ref); - input.reset(ref.bytes, ref.offset, ref.length); - currentOrd = 0; - } + } + + @Override + public void setDocument(int docID) { + docToOrds.get(docID, ref); + input.reset(ref.bytes, ref.offset, ref.length); + currentOrd = 0; + } - @Override - public void lookupOrd(long ord, BytesRef result) { - try { - in.setPosition(0); - fst.getFirstArc(firstArc); - IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts); - result.bytes = new byte[output.length]; - result.offset = 0; - result.length = 0; - Util.toBytesRef(output, result); - } catch (IOException bogus) { - throw new RuntimeException(bogus); - } + @Override + public void lookupOrd(long ord, BytesRef result) { + try { + in.setPosition(0); + fst.getFirstArc(firstArc); + IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts); + result.bytes = new byte[output.length]; + result.offset = 0; + result.length = 0; + Util.toBytesRef(output, result); + } catch (IOException bogus) { + throw new RuntimeException(bogus); } + } - @Override - public long lookupTerm(BytesRef key) { - try { - InputOutput o = fstEnum.seekCeil(key); - if (o == null) { - return -getValueCount()-1; - } else if (o.input.equals(key)) { - return o.output.intValue(); - } else { - return -o.output-1; - } - } catch (IOException bogus) { - throw new RuntimeException(bogus); + @Override + public long lookupTerm(BytesRef key) { + try { + InputOutput o = fstEnum.seekCeil(key); + if (o == null) { + return -getValueCount()-1; + } else if (o.input.equals(key)) { + return o.output.intValue(); + } else { + return -o.output-1; } + } catch (IOException bogus) { + throw new RuntimeException(bogus); } + } - @Override - public long getValueCount() { - return entry.numOrds; - } + @Override + public long getValueCount() { + return numOrds; + } - @Override - public TermsEnum termsEnum() { - return new FSTTermsEnum(fst); - } - }; + @Override + public TermsEnum termsEnum() { + return new FSTTermsEnum(fst); + } + + @Override + public SortedSetDocValues clone() { + return new MemorySortedSetDocValues(this); + } + } - + private Bits getMissingBits(int fieldNumber, final long offset, final long length) throws IOException { if (offset == -1) { return new Bits.MatchAllBits(maxDoc); Index: lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (revision 1516384) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (working copy) @@ -358,89 +358,128 @@ if (entry.numOrds == 0) { return SortedSetDocValues.EMPTY; // empty FST! } - FST instance; + FST fst; synchronized(this) { - instance = fstInstances.get(field.number); - if (instance == null) { + fst = fstInstances.get(field.number); + if (fst == null) { data.seek(entry.offset); - instance = new FST(data, PositiveIntOutputs.getSingleton()); - fstInstances.put(field.number, instance); + fst = new FST(data, PositiveIntOutputs.getSingleton()); + fstInstances.put(field.number, fst); } } final BinaryDocValues docToOrds = getBinary(field); - final FST fst = instance; - + + return new Lucene42SortedSetDocValues(docToOrds, fst, entry.numOrds); + } + + private static final class Lucene42SortedSetDocValues extends SortedSetDocValues { + + final BinaryDocValues docToOrds; + final FST fst; + final long numOrds; + // per-thread resources - final BytesReader in = fst.getBytesReader(); - final Arc firstArc = new Arc(); - final Arc scratchArc = new Arc(); - final IntsRef scratchInts = new IntsRef(); - final BytesRefFSTEnum fstEnum = new BytesRefFSTEnum(fst); - final BytesRef ref = new BytesRef(); - final ByteArrayDataInput input = new ByteArrayDataInput(); - return new SortedSetDocValues() { - long currentOrd; + final BytesReader in; + final Arc firstArc; + final Arc scratchArc; + final IntsRef scratchInts; + final BytesRefFSTEnum fstEnum; + final BytesRef ref; + final ByteArrayDataInput input; - @Override - public long nextOrd() { - if (input.eof()) { - return NO_MORE_ORDS; - } else { - currentOrd += input.readVLong(); - return currentOrd; - } + long currentOrd; + + Lucene42SortedSetDocValues(BinaryDocValues docToOrds, FST fst, long numOrds) { + this.docToOrds = docToOrds; + this.fst = fst; + this.numOrds = numOrds; + in = fst.getBytesReader(); + firstArc = new Arc(); + scratchArc = new Arc(); + scratchInts = new IntsRef(); + fstEnum = new BytesRefFSTEnum(fst); + ref = new BytesRef(); + input = new ByteArrayDataInput(); + } + + Lucene42SortedSetDocValues(Lucene42SortedSetDocValues other) { + docToOrds = other.docToOrds; + fst = other.fst; + numOrds = other.numOrds; + in = other.in; + firstArc = other.firstArc; + scratchArc = other.scratchArc; + scratchInts = other.scratchInts; + fstEnum = other.fstEnum; + ref = other.ref; + input = new ByteArrayDataInput(); + } + + @Override + public long nextOrd() { + if (input.eof()) { + return NO_MORE_ORDS; + } else { + currentOrd += input.readVLong(); + return currentOrd; } - - @Override - public void setDocument(int docID) { - docToOrds.get(docID, ref); - input.reset(ref.bytes, ref.offset, ref.length); - currentOrd = 0; - } + } + + @Override + public void setDocument(int docID) { + docToOrds.get(docID, ref); + input.reset(ref.bytes, ref.offset, ref.length); + currentOrd = 0; + } - @Override - public void lookupOrd(long ord, BytesRef result) { - try { - in.setPosition(0); - fst.getFirstArc(firstArc); - IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts); - result.bytes = new byte[output.length]; - result.offset = 0; - result.length = 0; - Util.toBytesRef(output, result); - } catch (IOException bogus) { - throw new RuntimeException(bogus); - } + @Override + public void lookupOrd(long ord, BytesRef result) { + try { + in.setPosition(0); + fst.getFirstArc(firstArc); + IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts); + result.bytes = new byte[output.length]; + result.offset = 0; + result.length = 0; + Util.toBytesRef(output, result); + } catch (IOException bogus) { + throw new RuntimeException(bogus); } + } - @Override - public long lookupTerm(BytesRef key) { - try { - InputOutput o = fstEnum.seekCeil(key); - if (o == null) { - return -getValueCount()-1; - } else if (o.input.equals(key)) { - return o.output.intValue(); - } else { - return -o.output-1; - } - } catch (IOException bogus) { - throw new RuntimeException(bogus); + @Override + public long lookupTerm(BytesRef key) { + try { + InputOutput o = fstEnum.seekCeil(key); + if (o == null) { + return -getValueCount()-1; + } else if (o.input.equals(key)) { + return o.output.intValue(); + } else { + return -o.output-1; } + } catch (IOException bogus) { + throw new RuntimeException(bogus); } + } - @Override - public long getValueCount() { - return entry.numOrds; - } + @Override + public long getValueCount() { + return numOrds; + } - @Override - public TermsEnum termsEnum() { - return new FSTTermsEnum(fst); - } - }; + @Override + public TermsEnum termsEnum() { + return new FSTTermsEnum(fst); + } + + @Override + public SortedSetDocValues clone() { + return new Lucene42SortedSetDocValues(this); + } + } - + @Override public Bits getDocsWithField(FieldInfo field) throws IOException { if (field.getDocValuesType() == FieldInfo.DocValuesType.SORTED_SET) { Index: lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java (revision 1516384) +++ lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java (working copy) @@ -515,5 +515,14 @@ public long getValueCount() { return mapping.getValueCount(); } + + @Override + public SortedSetDocValues clone() { + MultiSortedSetDocValues clone = (MultiSortedSetDocValues) super.clone(); + for (int i = 0; i < values.length; ++i) { + values[i] = values[i].clone(); + } + return clone; + } } } Index: lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java (revision 1516384) +++ lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java (working copy) @@ -26,8 +26,12 @@ * and sorted into a dictionary of unique values. A pointer to the * dictionary value (ordinal) can be retrieved for each document. Ordinals * are dense and in increasing sorted order. + *

+ * Expert: {@link #clone()} will return an instance that can be used in + * parallel with this instance in the same thread. This can be useful + * if you need to iterate over ordinals of two documents in parallel. */ -public abstract class SortedSetDocValues { +public abstract class SortedSetDocValues implements Cloneable { /** Sole constructor. (For invocation by subclass * constructors, typically implicit.) */ @@ -88,6 +92,11 @@ public long getValueCount() { return 0; } + + @Override + public SortedSetDocValues clone() { + return this; + } }; /** If {@code key} exists, returns its ordinal, else @@ -125,4 +134,17 @@ public TermsEnum termsEnum() { return new SortedSetDocValuesTermsEnum(this); } + + /** Expert: Return a clone of this instance, available for use in + * the current thread only. The returned instance will be in an undefined + * state so you need to call {@link #setDocument(int)} before calling + * {@link #nextOrd()}. */ + @Override + public SortedSetDocValues clone() { + try { + return (SortedSetDocValues) super.clone(); + } catch (CloneNotSupportedException e) { + throw new AssertionError(); + } + } } Index: lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (revision 1516384) +++ lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (working copy) @@ -774,7 +774,7 @@ } } - private class Iterator extends SortedSetDocValues { + private final class Iterator extends SortedSetDocValues { final AtomicReader reader; final TermsEnum te; // used internally for lookupOrd() and lookupTerm() // currently we read 5 at a time (using the logic of the old iterator) @@ -786,7 +786,7 @@ private int upto; private byte[] arr; - Iterator(AtomicReader reader) throws IOException { + Iterator(AtomicReader reader) { this.reader = reader; this.te = termsEnum(); } @@ -907,5 +907,10 @@ throw new RuntimeException(); } } + + @Override + public SortedSetDocValues clone() { + return new Iterator(reader); + } } } Index: lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (revision 1516384) +++ lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (working copy) @@ -280,7 +280,7 @@ dvFields.put(field, dvs); } - return dvs; + return dvs.clone(); } Bits getDocsWithField(String field) throws IOException {