Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SortingAtomicReaderTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SortingAtomicReaderTest.java (révision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SortingAtomicReaderTest.java (copie de travail) @@ -0,0 +1,75 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util._TestUtil; +import org.junit.BeforeClass; + +public class SortingAtomicReaderTest extends SorterTestBase { + + @BeforeClass + public static void beforeClassSortingAtomicReaderTest() throws Exception { + // build the mapping from the reader, since we deleted documents, some of + // them might have disappeared from the index (e.g. if an entire segment is + // dropped b/c all its docs are deleted) + Integer[] values = new Integer[reader.maxDoc()]; + int[] docs = new int[reader.maxDoc()]; + for (int i = 0; i < reader.maxDoc(); i++) { + docs[i] = i; + values[i] = Integer.valueOf(reader.document(i).get(ID_FIELD)); + } + + final int[] oldToNew = Sorter.compute(docs, Collections.unmodifiableList(Arrays.asList(values))); + // Sorter.compute also sorts the values + sortedValues = new Integer[reader.maxDoc()]; + for (int i = 0; i < reader.maxDoc(); ++i) { + sortedValues[oldToNew[i]] = values[i]; + } + if (VERBOSE) { + System.out.println("oldToNew: " + Arrays.toString(oldToNew)); + System.out.println("sortedValues: " + Arrays.toString(sortedValues)); + } + + reader = new SortingAtomicReader(reader, new Sorter() { + @Override + public int[] oldToNew(AtomicReader reader) throws IOException { + return oldToNew; + } + }); + + if (VERBOSE) { + System.out.print("mapped-deleted-docs: "); + Bits mappedLiveDocs = reader.getLiveDocs(); + for (int i = 0; i < mappedLiveDocs.length(); i++) { + if (!mappedLiveDocs.get(i)) { + System.out.print(i + " "); + } + } + System.out.println(); + } + + _TestUtil.checkReader(reader); + } + +} Modification de propriétés sur lucene/misc/src/test/org/apache/lucene/index/sorter/SortingAtomicReaderTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSorterTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSorterTest.java (révision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSorterTest.java (copie de travail) @@ -0,0 +1,89 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.StoredDocument; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util._TestUtil; +import org.junit.BeforeClass; + +public class IndexSorterTest extends SorterTestBase { + + private static final Sorter[] SORTERS = new Sorter[3]; + static { + SORTERS[0] = new StoredFieldsSorter() { + @Override + protected Integer parse(final StoredDocument doc) { + return Integer.valueOf(doc.getField(ID_FIELD).stringValue()); + } + }; + SORTERS[1] = new NumericDocValuesSorter(NUMERIC_DV_FIELD); + SORTERS[2] = new PayloadSorter(new Term(PAYLOAD_FIELD, PAYLOAD_TERM)) { + @Override + protected Integer parse(final BytesRef bytes) { + return Integer.valueOf(bytes.utf8ToString()); + } + }; + } + + @BeforeClass + public static void beforeClassSorterUtilTest() throws Exception { + // only read the values of the undeleted documents, since after SorterUtil, + // the deleted ones will be dropped from the index. + Bits liveDocs = reader.getLiveDocs(); + List values = new ArrayList(); + for (int i = 0; i < reader.maxDoc(); i++) { + if (liveDocs == null || liveDocs.get(i)) { + values.add(Integer.valueOf(reader.document(i).get(ID_FIELD))); + } + } + Collections.sort(values); + sortedValues = values.toArray(new Integer[values.size()]); + if (VERBOSE) { + System.out.println("sortedValues: " + sortedValues); + } + + // close the reader; will open a new one on the sorted index + reader.close(); + Directory target = newDirectory(); + Sorter sorter = SORTERS[random().nextInt(SORTERS.length)]; + if (VERBOSE) { + System.out.println("sorter: " + sorter); + } + IndexSorter.sort(dir, target, sorter); + dir.close(); + dir = target; + + // CheckIndex the target directory + _TestUtil.checkIndex(target); + + // set reader for tests + reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir)); + assertFalse("index should not have deletions", reader.hasDeletions()); + } + +} Modification de propriétés sur lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSorterTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (révision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (copie de travail) @@ -0,0 +1,413 @@ +package org.apache.lucene.index.sorter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.CollectionStatistics; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public abstract class SorterTestBase extends LuceneTestCase { + + static final class NormsSimilarity extends Similarity { + + private final Similarity in; + + public NormsSimilarity(Similarity in) { + this.in = in; + } + + @Override + public long computeNorm(FieldInvertState state) { + if (state.getName().equals(NORMS_FIELD)) { + return Float.floatToIntBits(state.getBoost()); + } else { + return in.computeNorm(state); + } + } + + @Override + public SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) { + return in.computeWeight(queryBoost, collectionStats, termStats); + } + + @Override + public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException { + return in.exactSimScorer(weight, context); + } + + @Override + public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException { + return in.sloppySimScorer(weight, context); + } + + } + + static final class PositionsTokenStream extends TokenStream { + + private final CharTermAttribute term; + private final PayloadAttribute payload; + private final OffsetAttribute offset; + + private int pos, off; + + public PositionsTokenStream() { + term = addAttribute(CharTermAttribute.class); + term.append(DOC_POSITIONS_TERM); + payload = addAttribute(PayloadAttribute.class); + offset = addAttribute(OffsetAttribute.class); + } + + @Override + public boolean incrementToken() throws IOException { + if (pos == 0) { + return false; + } + + payload.setPayload(new BytesRef(Integer.toString(pos))); + offset.setOffset(off, off); + --pos; + ++off; + return true; + } + + void setId(int id) { + pos = id / 10 + 1; + off = 0; + } + } + + static final class PayloadTokenStream extends TokenStream { + + private final CharTermAttribute term; + private final PayloadAttribute payload; + + private boolean done = false; + private int id; + + public PayloadTokenStream() { + term = addAttribute(CharTermAttribute.class); + term.append(PAYLOAD_TERM); + payload = addAttribute(PayloadAttribute.class); + } + + @Override + public boolean incrementToken() throws IOException { + if (done) { + return false; + } + + payload.setPayload(new BytesRef(Integer.toString(id))); + done = true; + return true; + } + + void setId(int id) { + this.id = id; + done = false; + } + } + + protected static final String ID_FIELD = "id"; + protected static final String DOCS_ENUM_FIELD = "docs"; + protected static final String DOCS_ENUM_TERM = "$all$"; + protected static final String DOC_POSITIONS_FIELD = "positions"; + protected static final String DOC_POSITIONS_TERM = "$all$"; + protected static final String NUMERIC_DV_FIELD = "numeric"; + protected static final String NORMS_FIELD = "norm"; + protected static final String BINARY_DV_FIELD = "binary"; + protected static final String SORTED_DV_FIELD = "sorted"; + protected static final String SORTED_SET_DV_FIELD = "sorted_set"; + protected static final String TERM_VECTORS_FIELD = "term_vectors"; + protected static final String PAYLOAD_FIELD = "payload"; + protected static final String PAYLOAD_TERM = "$all$"; + + private static final FieldType TERM_VECTORS_TYPE = new FieldType(TextField.TYPE_NOT_STORED); + static { + TERM_VECTORS_TYPE.setStoreTermVectors(true); + TERM_VECTORS_TYPE.freeze(); + } + + private static final FieldType POSITIONS_TYPE = new FieldType(TextField.TYPE_NOT_STORED); + static { + POSITIONS_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + POSITIONS_TYPE.freeze(); + } + + protected static Directory dir; + protected static AtomicReader reader; + protected static Integer[] sortedValues; + + private static Document doc(final int id, PositionsTokenStream positions, PayloadTokenStream payloads) { + final Document doc = new Document(); + doc.add(new StringField(ID_FIELD, Integer.toString(id), Store.YES)); + doc.add(new StringField(DOCS_ENUM_FIELD, DOCS_ENUM_TERM, Store.NO)); + positions.setId(id); + if (doesntSupportOffsets.contains(_TestUtil.getPostingsFormat(DOC_POSITIONS_FIELD))) { + // codec doesnt support offsets: just index positions for the field + doc.add(new Field(DOC_POSITIONS_FIELD, positions, TextField.TYPE_NOT_STORED)); + } else { + doc.add(new Field(DOC_POSITIONS_FIELD, positions, POSITIONS_TYPE)); + } + payloads.setId(id); + doc.add(new TextField(PAYLOAD_FIELD, payloads)); + doc.add(new NumericDocValuesField(NUMERIC_DV_FIELD, id)); + TextField norms = new TextField(NORMS_FIELD, Integer.toString(id), Store.NO); + norms.setBoost(Float.intBitsToFloat(id)); + doc.add(norms); + doc.add(new BinaryDocValuesField(BINARY_DV_FIELD, new BytesRef(Integer.toString(id)))); + doc.add(new SortedDocValuesField(SORTED_DV_FIELD, new BytesRef(Integer.toString(id)))); + if (defaultCodecSupportsSortedSet()) { + doc.add(new SortedSetDocValuesField(SORTED_SET_DV_FIELD, new BytesRef(Integer.toString(id)))); + doc.add(new SortedSetDocValuesField(SORTED_SET_DV_FIELD, new BytesRef(Integer.toString(id + 1)))); + } + doc.add(new Field(TERM_VECTORS_FIELD, Integer.toString(id), TERM_VECTORS_TYPE)); + return doc; + } + + /** Creates an index for sorting. */ + public static void createIndex(Directory dir, int numDocs, Random random) throws IOException { + List ids = new ArrayList(); + for (int i = 0; i < numDocs; i++) { + ids.add(Integer.valueOf(i * 10)); + } + // shuffle them for indexing + Collections.shuffle(ids, random); + if (VERBOSE) { + System.out.println("Shuffled IDs for indexing: " + Arrays.toString(ids.toArray())); + } + + PositionsTokenStream positions = new PositionsTokenStream(); + PayloadTokenStream payloads = new PayloadTokenStream(); + IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); + conf.setMaxBufferedDocs(4); // create some segments + conf.setSimilarity(new NormsSimilarity(conf.getSimilarity())); // for testing norms field + RandomIndexWriter writer = new RandomIndexWriter(random, dir, conf); + writer.setDoRandomForceMerge(false); + for (int id : ids) { + writer.addDocument(doc(id, positions, payloads)); + } + // delete some documents + writer.commit(); + for (Integer id : ids) { + if (random.nextDouble() < 0.2) { + if (VERBOSE) { + System.out.println("delete doc_id " + id); + } + writer.deleteDocuments(new Term(ID_FIELD, id.toString())); + } + } + writer.close(); + } + + @BeforeClass + public static void beforeClassSorterTestBase() throws Exception { + dir = newDirectory(); + int numDocs = atLeast(20); + createIndex(dir, numDocs, random()); + + reader = new SlowCompositeReaderWrapper(DirectoryReader.open(dir)); + } + + @AfterClass + public static void afterClassSorterTestBase() throws Exception { + reader.close(); + dir.close(); + } + + @Test + public void testBinaryDocValuesField() throws Exception { + BinaryDocValues dv = reader.getBinaryDocValues(BINARY_DV_FIELD); + BytesRef bytes = new BytesRef(); + for (int i = 0; i < reader.maxDoc(); i++) { + dv.get(i, bytes); + assertEquals("incorrect binary DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString()); + } + } + + @Test + public void testDocsAndPositionsEnum() throws Exception { + Term term = new Term(DOC_POSITIONS_FIELD, DOC_POSITIONS_TERM); + DocsAndPositionsEnum sortedPositions = reader.termPositionsEnum(term); + int doc; + + // test nextDoc() + while ((doc = sortedPositions.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + int freq = sortedPositions.freq(); + assertEquals("incorrect freq for doc=" + doc, sortedValues[doc].intValue() / 10 + 1, freq); + for (int i = 0; i < freq; i++) { + assertEquals("incorrect position for doc=" + doc, i, sortedPositions.nextPosition()); + if (!doesntSupportOffsets.contains(_TestUtil.getPostingsFormat(DOC_POSITIONS_FIELD))) { + assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.startOffset()); + assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.endOffset()); + } + assertEquals("incorrect payload for doc=" + doc, freq - i, Integer.parseInt(sortedPositions.getPayload().utf8ToString())); + } + } + + // test advance() + sortedPositions = reader.termPositionsEnum(term); + doc = 0; + while ((doc = sortedPositions.advance(doc)) != DocIdSetIterator.NO_MORE_DOCS) { + int freq = sortedPositions.freq(); + assertEquals("incorrect freq for doc=" + doc, sortedValues[doc].intValue() / 10 + 1, freq); + for (int i = 0; i < freq; i++) { + assertEquals("incorrect position for doc=" + doc, i, sortedPositions.nextPosition()); + if (!doesntSupportOffsets.contains(_TestUtil.getPostingsFormat(DOC_POSITIONS_FIELD))) { + assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.startOffset()); + assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.endOffset()); + } + assertEquals("incorrect payload for doc=" + doc, freq - i, Integer.parseInt(sortedPositions.getPayload().utf8ToString())); + } + } + } + + @Test + public void testDocsEnum() throws Exception { + Term term = new Term(DOCS_ENUM_FIELD, DOCS_ENUM_TERM); + DocsEnum docs = reader.termDocsEnum(term); + Bits mappedLiveDocs = reader.getLiveDocs(); + int doc; + int prev = -1; + while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (mappedLiveDocs != null) { + assertTrue("document " + doc + " marked as deleted", mappedLiveDocs.get(doc)); + } + assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(reader.document(doc).get(ID_FIELD))); + while (++prev < doc) { + assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs.get(prev)); + } + } + + docs = reader.termDocsEnum(term); + doc = 0; + prev = -1; + while ((doc = docs.advance(doc)) != DocIdSetIterator.NO_MORE_DOCS) { + if (mappedLiveDocs != null) { + assertTrue("document " + doc + " marked as deleted", mappedLiveDocs.get(doc)); + } + assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(reader.document(doc).get(ID_FIELD))); + while (++prev < doc) { + assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs.get(prev)); + } + } + } + + @Test + public void testNormValues() throws Exception { + NumericDocValues dv = reader.getNormValues(NORMS_FIELD); + int maxDoc = reader.maxDoc(); + for (int i = 0; i < maxDoc; i++) { + assertEquals("incorrect norm value for doc " + i, sortedValues[i].intValue(), dv.get(i)); + } + } + + @Test + public void testNumericDocValuesField() throws Exception { + NumericDocValues dv = reader.getNumericDocValues(NUMERIC_DV_FIELD); + int maxDoc = reader.maxDoc(); + for (int i = 0; i < maxDoc; i++) { + assertEquals("incorrect numeric DocValues for doc " + i, sortedValues[i].intValue(), dv.get(i)); + } + } + + @Test + public void testSortedDocValuesField() throws Exception { + SortedDocValues dv = reader.getSortedDocValues(SORTED_DV_FIELD); + int maxDoc = reader.maxDoc(); + BytesRef bytes = new BytesRef(); + for (int i = 0; i < maxDoc; i++) { + dv.get(i, bytes); + assertEquals("incorrect sorted DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString()); + } + } + + @Test + public void testSortedSetDocValuesField() throws Exception { + assumeTrue("default codec does not support SORTED_SET", defaultCodecSupportsSortedSet()); + SortedSetDocValues dv = reader.getSortedSetDocValues(SORTED_SET_DV_FIELD); + int maxDoc = reader.maxDoc(); + BytesRef bytes = new BytesRef(); + for (int i = 0; i < maxDoc; i++) { + dv.setDocument(i); + dv.lookupOrd(dv.nextOrd(), bytes); + int value = sortedValues[i].intValue(); + assertEquals("incorrect sorted-set DocValues for doc " + i, Integer.valueOf(value).toString(), bytes.utf8ToString()); + dv.lookupOrd(dv.nextOrd(), bytes); + assertEquals("incorrect sorted-set DocValues for doc " + i, Integer.valueOf(value + 1).toString(), bytes.utf8ToString()); + assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd()); + } + } + + @Test + public void testTermVectors() throws Exception { + int maxDoc = reader.maxDoc(); + for (int i = 0; i < maxDoc; i++) { + Terms terms = reader.getTermVector(i, TERM_VECTORS_FIELD); + assertNotNull("term vectors not found for doc " + i + " field [" + TERM_VECTORS_FIELD + "]", terms); + assertEquals("incorrect term vector for doc " + i, sortedValues[i].toString(), terms.iterator(null).next().utf8ToString()); + } + } + +} Modification de propriétés sur lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTest.java (révision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTest.java (copie de travail) @@ -0,0 +1,174 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.StoredDocument; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.sorter.SorterTestBase.PayloadTokenStream; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class SorterTest extends LuceneTestCase { + + private static final String ID_FIELD = "id"; + private static final String PAYLOAD_FIELD = "payload"; + private static final String PAYLOAD_TERM = "$all$"; + private static final String NUMERIC_DV_FIELD = "numeric"; + + private static Directory dir; + private static PayloadTokenStream payloads; + private static AtomicReader reader; + private static int[] expOldToNew; + + private static Document doc(final int id) { + final Document doc = new Document(); + doc.add(new StoredField(ID_FIELD, id)); + payloads.setId(id); + doc.add(new TextField(PAYLOAD_FIELD, payloads)); + doc.add(new NumericDocValuesField(NUMERIC_DV_FIELD, id)); + return doc; + } + + @BeforeClass + public static void beforeClassSorterTest() throws Exception { + payloads = new PayloadTokenStream(); + dir = newDirectory(); + int numDocs = atLeast(20); + List ids = new ArrayList(); + for (int i = 0; i < numDocs; i++) { + ids.add(i * 10); + } + Collections.shuffle(ids, random()); + int[] docs = new int[numDocs]; + Integer[] values = new Integer[numDocs]; + int idx = 0; + for (Integer id : ids) { + docs[idx] = idx; + values[idx++] = id; + } + expOldToNew = Sorter.compute(docs, Collections.unmodifiableList(Arrays.asList(values))); + + // use LogMergePolicy because we rely on docs order + IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + conf.setMaxBufferedDocs(4); + conf.setMergePolicy(NoMergePolicy.COMPOUND_FILES); // no merges + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf); + writer.setDoRandomForceMerge(false); + for (Integer id : ids) { + writer.addDocument(doc(id)); + } + writer.close(); + + if (VERBOSE) { + System.out.println("Shuffled IDs: " + ids); + System.out.println("expOldToNew: " + Arrays.toString(expOldToNew)); + } + + reader = new SlowCompositeReaderWrapper(DirectoryReader.open(dir)); + } + + @AfterClass + public static void afterClassSorterTest() throws Exception { + reader.close(); + dir.close(); + } + + @Test + public void testPayload() throws IOException { + Term term = new Term(PAYLOAD_FIELD, PAYLOAD_TERM); + final PayloadSorter sorter = new PayloadSorter(term) { + @Override + protected Integer parse(final BytesRef bytes) { + return Integer.valueOf(bytes.utf8ToString()); + } + }; + + DirectoryReader r = DirectoryReader.open(dir); + try { + assertArrayEquals(expOldToNew, sorter.oldToNew(SlowCompositeReaderWrapper.wrap(r))); + } finally { + r.close(); + } + } + + @Test + public void testNumericDocValues() throws IOException { + final NumericDocValuesSorter sorter = new NumericDocValuesSorter(NUMERIC_DV_FIELD); + DirectoryReader r = DirectoryReader.open(dir); + try { + assertArrayEquals(expOldToNew, sorter.oldToNew(SlowCompositeReaderWrapper.wrap(r))); + } finally { + r.close(); + } + } + + @Test + public void testStoredFields() throws IOException { + final StoredFieldsSorter sorter = new StoredFieldsSorter() { + + @Override + protected Integer parse(final StoredDocument doc) { + return Integer.valueOf(doc.getField(ID_FIELD).numericValue().intValue()); + } + }; + + DirectoryReader r = DirectoryReader.open(dir); + try { + assertArrayEquals(expOldToNew, sorter.oldToNew(SlowCompositeReaderWrapper.wrap(r))); + } finally { + r.close(); + } + } + + @Test + public void testReverseDocId() throws IOException { + final ReverseDocIdSorter sorter = new ReverseDocIdSorter(); + DirectoryReader r = DirectoryReader.open(dir); + try { + int[] expReverse = new int[expOldToNew.length]; + for (int i = 0; i < expReverse.length; i++) { + expReverse[expReverse.length - i - 1] = i; + } + assertArrayEquals(expReverse, sorter.oldToNew(SlowCompositeReaderWrapper.wrap(r))); + } finally { + r.close(); + } + } + +} Modification de propriétés sur lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (copie de travail) @@ -0,0 +1,624 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.FilterAtomicReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMFile; +import org.apache.lucene.store.RAMInputStream; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.SorterTemplate; + +/** + * An {@link AtomicReader} which supports sorting documents by a given + * {@link Sorter}. + *

+ * NOTE: currently this reader does not support input {@link IndexReader + * readers} with deleted documents. + * + * @lucene.experimental + */ +public class SortingAtomicReader extends FilterAtomicReader { + + private static class SortingFields extends FilterFields { + + private final int[] old2new; + private final Bits inLiveDocs; + + public SortingFields(final Fields in, final Bits inLiveDocs, final int[] old2new) { + super(in); + this.old2new = old2new; + this.inLiveDocs = inLiveDocs; + } + + @Override + public Terms terms(final String field) throws IOException { + Terms terms = in.terms(field); + if (terms == null) { + return null; + } else { + return new SortingTerms(terms, inLiveDocs, old2new); + } + } + + } + + private static class SortingTerms extends FilterTerms { + + private final int[] old2new; + private final Bits inLiveDocs; + + public SortingTerms(final Terms in, final Bits inLiveDocs, final int[] old2new) { + super(in); + this.old2new = old2new; + this.inLiveDocs = inLiveDocs; + } + + @Override + public TermsEnum iterator(final TermsEnum reuse) throws IOException { + return new SortingTermsEnum(in.iterator(reuse), inLiveDocs, old2new); + } + + } + + private static class SortingTermsEnum extends FilterTermsEnum { + + private final int[] old2new; + private final Bits inLiveDocs; + + public SortingTermsEnum(final TermsEnum in, final Bits inLiveDocs, final int[] old2new) { + super(in); + this.old2new = old2new; + this.inLiveDocs = inLiveDocs; + } + + @Override + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, final int flags) throws IOException { + if (liveDocs != null) { + liveDocs = inLiveDocs; + } + + // if we're asked to reuse the given DocsEnum and it is Sorting, return + // the wrapped one, since some Codecs expect it. + if (reuse != null && reuse instanceof SortingDocsEnum) { + reuse = ((SortingDocsEnum) reuse).getWrapped(); + } + return new SortingDocsEnum(in.docs(liveDocs, reuse, flags), flags, old2new); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, final int flags) throws IOException { + if (liveDocs != null) { + liveDocs = inLiveDocs; + } + + // if we're asked to reuse the given DocsAndPositionsEnum and it is + // Sorting, return the wrapped one, since some Codecs expect it. + if (reuse != null && reuse instanceof SortingDocsAndPositionsEnum) { + reuse = ((SortingDocsAndPositionsEnum) reuse).getWrapped(); + } + + final DocsAndPositionsEnum positions = in.docsAndPositions(liveDocs, reuse, flags); + if (positions == null) { + return null; + } else { + return new SortingDocsAndPositionsEnum(positions, old2new); + } + } + + } + + private static class SortingBinaryDocValues extends BinaryDocValues { + + private final BinaryDocValues in; + private final int[] new2old; + + SortingBinaryDocValues(BinaryDocValues in, int[] new2old) { + this.in = in; + this.new2old = new2old; + } + + @Override + public void get(int docID, BytesRef result) { + in.get(new2old[docID], result); + } + } + + private static class SortingNumericDocValues extends NumericDocValues { + + private final NumericDocValues in; + private final int[] new2old; + + public SortingNumericDocValues(final NumericDocValues in, final int[] new2old) { + this.in = in; + this.new2old = new2old; + } + + @Override + public long get(int docID) { + return in.get(new2old[docID]); + } + } + + private static class SortingSortedDocValues extends SortedDocValues { + + private final SortedDocValues in; + private final int[] new2old; + + SortingSortedDocValues(SortedDocValues in, int[] new2old) { + this.in = in; + this.new2old = new2old; + } + + @Override + public int getOrd(int docID) { + return in.getOrd(new2old[docID]); + } + + @Override + public void lookupOrd(int ord, BytesRef result) { + in.lookupOrd(ord, result); + } + + @Override + public int getValueCount() { + return in.getValueCount(); + } + + @Override + public void get(int docID, BytesRef result) { + in.get(new2old[docID], result); + } + + @Override + public int lookupTerm(BytesRef key) { + return in.lookupTerm(key); + } + } + + private static class SortingSortedSetDocValues extends SortedSetDocValues { + + private final SortedSetDocValues in; + private final int[] new2old; + + SortingSortedSetDocValues(SortedSetDocValues in, int[] new2old) { + this.in = in; + this.new2old = new2old; + } + + @Override + public long nextOrd() { + return in.nextOrd(); + } + + @Override + public void setDocument(int docID) { + in.setDocument(new2old[docID]); + } + + @Override + public void lookupOrd(long ord, BytesRef result) { + in.lookupOrd(ord, result); + } + + @Override + public long getValueCount() { + return in.getValueCount(); + } + + @Override + public long lookupTerm(BytesRef key) { + return in.lookupTerm(key); + } + } + + private static class SortingDocsEnum extends FilterDocsEnum { + + private static final class DocFreqSorterTemplate extends SorterTemplate { + + private final int[] docs; + private final int[] freqs; + + private int pivot; + + public DocFreqSorterTemplate(int[] docs, int[] freqs) { + this.docs = docs; + this.freqs = freqs; + } + + @Override + protected int compare(int i, int j) { + return docs[i] - docs[j]; + } + + @Override + protected int comparePivot(int j) { + return pivot - docs[j]; + } + + @Override + protected void setPivot(int i) { + pivot = docs[i]; + } + + @Override + protected void swap(int i, int j) { + int tmpDoc = docs[i]; + docs[i] = docs[j]; + docs[j] = tmpDoc; + + int tmpFreq = freqs[i]; + freqs[i] = freqs[j]; + freqs[j] = tmpFreq; + } + } + + private int[] docs = new int[64]; + private int[] freqs; + private int docIt = -1; + private final int upto; + private final boolean withFreqs; + + public SortingDocsEnum(final DocsEnum in, int flags, final int[] old2new) throws IOException { + super(in); + withFreqs = (flags & DocsEnum.FLAG_FREQS) != 0; + int i = 0; + int doc; + if (withFreqs) { + freqs = new int[docs.length]; + while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){ + if (i >= docs.length) { + docs = ArrayUtil.grow(docs, docs.length + 1); + freqs = ArrayUtil.grow(freqs, freqs.length + 1); + } + docs[i] = old2new[doc]; + freqs[i] = in.freq(); + ++i; + } + SorterTemplate sorter = new DocFreqSorterTemplate(docs, freqs); + sorter.quickSort(0, i - 1); + } else { + freqs = null; + while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){ + if (i >= docs.length) { + docs = ArrayUtil.grow(docs, docs.length + 1); + } + docs[i++] = old2new[doc]; + } + Arrays.sort(docs, 0, i); + } + upto = i; + } + + @Override + public int advance(final int target) throws IOException { + // need to support it for checkIndex, but in practice it won't be called, so + // don't bother to implement efficiently for now. + while (nextDoc() < target) {} + return docID(); + } + + @Override + public int docID() { + return docIt >= upto ? NO_MORE_DOCS : docs[docIt]; + } + + @Override + public int freq() throws IOException { + return withFreqs && docIt < upto ? freqs[docIt] : 1; + } + + @Override + public int nextDoc() throws IOException { + if (++docIt >= upto) return NO_MORE_DOCS; + return docs[docIt]; + } + + /** Returns the wrapped {@link DocsEnum}. */ + DocsEnum getWrapped() { + return in; + } + } + + private static class SortingDocsAndPositionsEnum extends FilterDocsAndPositionsEnum { + + /** + * A {@link SorterTemplate} which sorts two parallel arrays of doc IDs and + * offsets in one go. Everytime a doc ID is 'swapped', its correponding offset + * is swapped too. + */ + private static final class DocOffsetSorterTemplate extends SorterTemplate { + + private final int[] docs; + private final long[] offsets; + + private int pivot; + + public DocOffsetSorterTemplate(int[] docs, long[] offsets) { + this.docs = docs; + this.offsets = offsets; + } + + @Override + protected int compare(int i, int j) { + return docs[i] - docs[j]; + } + + @Override + protected int comparePivot(int j) { + return pivot - docs[j]; + } + + @Override + protected void setPivot(int i) { + pivot = docs[i]; + } + + @Override + protected void swap(int i, int j) { + int tmpDoc = docs[i]; + docs[i] = docs[j]; + docs[j] = tmpDoc; + + long tmpOffset = offsets[i]; + offsets[i] = offsets[j]; + offsets[j] = tmpOffset; + } + } + + private int[] docs; + private long[] offsets; + private final int upto; + + private final IndexInput postingInput; + + private int docIt = -1; + private int pos; + private int startOffset; + private int endOffset; + private final BytesRef payload = new BytesRef(32); + private int currFreq; + + public SortingDocsAndPositionsEnum(final DocsAndPositionsEnum in, final int[] old2new) throws IOException { + super(in); + final RAMFile file = new RAMFile(); + final IndexOutput out = new RAMOutputStream(file); + docs = new int[32]; + offsets = new long[32]; + int doc; + int i = 0; + while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (i == docs.length) { + docs = ArrayUtil.grow(docs, i + 1); + offsets = ArrayUtil.grow(offsets, i + 1); + } + docs[i] = old2new[doc]; + offsets[i] = out.getFilePointer(); + addPositions(in, out); + i++; + } + upto = i; + SorterTemplate sorter = new DocOffsetSorterTemplate(docs, offsets); + sorter.quickSort(0, upto - 1); + out.close(); + this.postingInput = new RAMInputStream("", file); + } + + private void addPositions(final DocsAndPositionsEnum in, final IndexOutput out) throws IOException { + int freq = in.freq(); + out.writeVInt(freq); + for (int i = 0; i < freq; i++) { + final int oldPos = in.nextPosition(); + + out.writeVInt(oldPos); + out.writeVInt(in.startOffset()); + out.writeVInt(in.endOffset()); + + BytesRef payload = in.getPayload(); + if (payload != null) { + out.writeVInt(payload.length); + out.writeBytes(payload.bytes, payload.offset, payload.length); + } else { + out.writeVInt(0); + } + } + } + + @Override + public int advance(final int target) throws IOException { + // need to support it for checkIndex, but in practice it won't be called, so + // don't bother to implement efficiently for now. + while (nextDoc() < target) {} + return docID(); + } + + @Override + public int docID() { + return docIt >= upto ? NO_MORE_DOCS : docs[docIt]; + } + + @Override + public int endOffset() throws IOException { + return endOffset; + } + + @Override + public int freq() throws IOException { + return currFreq; + } + + @Override + public BytesRef getPayload() throws IOException { + return payload.length == 0 ? null : payload; + } + + @Override + public int nextDoc() throws IOException { + if (++docIt >= upto) return DocIdSetIterator.NO_MORE_DOCS; + postingInput.seek(offsets[docIt]); + currFreq = postingInput.readVInt(); + return docs[docIt]; + } + + @Override + public int nextPosition() throws IOException { + pos = postingInput.readVInt(); + startOffset = postingInput.readVInt(); + endOffset = postingInput.readVInt(); + int length = postingInput.readVInt(); + if (length > 0) { + if (length >= payload.bytes.length) { + payload.grow(length + 1); + } + postingInput.readBytes(payload.bytes, 0, length); + } + payload.length = length; + return pos; + } + + @Override + public int startOffset() throws IOException { + return startOffset; + } + + /** Returns the wrapped {@link DocsAndPositionsEnum}. */ + DocsAndPositionsEnum getWrapped() { + return in; + } + } + + private final int[] old2new, new2old; + private final FixedBitSet mappedLiveDocs; + + public SortingAtomicReader(final AtomicReader in, final Sorter sorter) throws IOException { + super(in); + old2new = sorter.oldToNew(in); + new2old = new int[old2new.length]; + for (int i = 0; i < new2old.length; i++) { + new2old[old2new[i]] = i; + } + + if (!in.hasDeletions()) { + mappedLiveDocs = null; + } else { + mappedLiveDocs = new FixedBitSet(in.maxDoc()); + mappedLiveDocs.set(0, in.maxDoc()); + Bits liveDocs = in.getLiveDocs(); + int len = liveDocs.length(); + for (int i = 0; i < len; i++) { + if (!liveDocs.get(i)) { + mappedLiveDocs.clear(old2new[i]); + } + } + } + } + + @Override + public void document(final int docID, final StoredFieldVisitor visitor) throws IOException { + in.document(new2old[docID], visitor); + } + + @Override + public Fields fields() throws IOException { + Fields fields = in.fields(); + if (fields == null) { + return null; + } else { + return new SortingFields(fields, in.getLiveDocs(), old2new); + } + } + + @Override + public BinaryDocValues getBinaryDocValues(String field) throws IOException { + BinaryDocValues oldDocValues = in.getBinaryDocValues(field); + if (oldDocValues == null) { + return null; + } else { + return new SortingBinaryDocValues(oldDocValues, new2old); + } + } + + @Override + public Bits getLiveDocs() { + ensureOpen(); + return mappedLiveDocs; + } + + @Override + public NumericDocValues getNormValues(String field) throws IOException { + final NumericDocValues norm = in.getNormValues(field); + if (norm == null) { + return null; + } else { + return new SortingNumericDocValues(norm, new2old); + } + } + + @Override + public NumericDocValues getNumericDocValues(String field) throws IOException { + final NumericDocValues oldDocValues = in.getNumericDocValues(field); + if (oldDocValues == null) return null; + return new SortingNumericDocValues(oldDocValues, new2old); + } + + @Override + public SortedDocValues getSortedDocValues(String field) throws IOException { + SortedDocValues sortedDV = in.getSortedDocValues(field); + if (sortedDV == null) { + return null; + } else { + return new SortingSortedDocValues(sortedDV, new2old); + } + } + + @Override + public SortedSetDocValues getSortedSetDocValues(String field) throws IOException { + SortedSetDocValues sortedSetDV = in.getSortedSetDocValues(field); + if (sortedSetDV == null) { + return null; + } else { + return new SortingSortedSetDocValues(sortedSetDV, new2old); + } + } + + @Override + public Fields getTermVectors(final int docID) throws IOException { + return in.getTermVectors(new2old[docID]); + } + +} Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/IndexSorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/IndexSorter.java (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/IndexSorter.java (copie de travail) @@ -0,0 +1,58 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.Version; + +/*** + * A utility class for sorting an index. Takes an input and output + * {@link Directory} and sorts the input index by calling + * {@link IndexWriter#addIndexes(org.apache.lucene.index.IndexReader...)} on the + * target one. If you cannot pass an input/output directories, use + * {@link SortingAtomicReader} directly. + * + * @lucene.experimental + */ +public class IndexSorter { + + /** + * Sorts the index in the input {@link Directory} and writes the sorted + * version to {@code out}. + */ + public static void sort(Directory in, Directory out, Sorter sorter) throws IOException { + IndexWriter writer = null; + DirectoryReader reader = null; + SortingAtomicReader sortingReader = null; + try { + writer = new IndexWriter(out, new IndexWriterConfig(Version.LUCENE_50, null)); + reader = DirectoryReader.open(in); + sortingReader = new SortingAtomicReader(SlowCompositeReaderWrapper.wrap(reader), sorter); + writer.addIndexes(sortingReader); + } finally { + IOUtils.close(writer, reader, sortingReader); + } + } + +} Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/IndexSorter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/NumericDocValuesSorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/NumericDocValuesSorter.java (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/NumericDocValuesSorter.java (copie de travail) @@ -0,0 +1,65 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.AbstractList; +import java.util.List; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.NumericDocValues; + +/** + * A {@link Sorter} which sorts documents according to their + * {@link NumericDocValues}. + * + * @lucene.experimental + */ +public class NumericDocValuesSorter extends Sorter { + + private final String fieldName; + + public NumericDocValuesSorter(final String fieldName) { + this.fieldName = fieldName; + } + + @Override + public int[] oldToNew(final AtomicReader reader) throws IOException { + final NumericDocValues ndv = reader.getNumericDocValues(fieldName); + final int maxDoc = reader.maxDoc(); + final int[] docs = new int[maxDoc]; + final List values = new AbstractList() { + + @Override + public Long get(int doc) { + return ndv.get(doc); + } + + @Override + public int size() { + return reader.maxDoc(); + } + + }; + for (int i = 0; i < maxDoc; i++) { + docs[i] = i; + } + return compute(docs, values); + } + +} Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/NumericDocValuesSorter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/ReverseDocIdSorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/ReverseDocIdSorter.java (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/ReverseDocIdSorter.java (copie de travail) @@ -0,0 +1,41 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.AtomicReader; + +/** + * A {@link Sorter} which sorts document in a reverse order to their doc id's + * + * @lucene.experimental + */ +public class ReverseDocIdSorter extends Sorter { + + @Override + public int[] oldToNew(final AtomicReader reader) throws IOException { + final int maxDoc = reader.maxDoc(); + int[] reverseDocs = new int[maxDoc]; + for (int i = 0; i < maxDoc; i++) { + reverseDocs[i] = maxDoc - (i + 1); + } + return reverseDocs; + } + +} Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/ReverseDocIdSorter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/package.html =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/package.html (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/package.html (copie de travail) @@ -0,0 +1,27 @@ + + + + +Provides index sorting capablities. The application can use one of the +pre-existing Sorter implementations, e.g. to sort by a stored field, a numeric +DocValues, a payload or reverse the order of the documents. Additionally, the +application can implement the Sorter interface which returns a permutation on +a source Directory's document IDs, to sort the input documents by additional +values. + + Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/package.html ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (copie de travail) @@ -0,0 +1,93 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.List; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.util.SorterTemplate; + +/** + * Sorts documents in a given index by returning a permutation on the docs. + * Implementations can call {@link #compute(int[], List)} to compute the + * old-to-new permutation over the given documents and values. + * + * @lucene.experimental + */ +public abstract class Sorter { + + public static final class DocValueSorterTemplate> extends SorterTemplate { + + private final int[] docs; + private final List values; + + private T pivot; + + public DocValueSorterTemplate(int[] docs, List values) { + this.docs = docs; + this.values = values; + } + + @Override + protected int compare(int i, int j) { + return values.get(docs[i]).compareTo(values.get(docs[j])); + } + + @Override + protected int comparePivot(int j) { + return pivot.compareTo(values.get(docs[j])); + } + + @Override + protected void setPivot(int i) { + pivot = values.get(docs[i]); + } + + @Override + protected void swap(int i, int j) { + int tmpDoc = docs[i]; + docs[i] = docs[j]; + docs[j] = tmpDoc; + } + } + + /** Computes the old-to-new permutation over the given documents and values. */ + protected static > int[] compute(int[] docs, List values) { + SorterTemplate sorter = new DocValueSorterTemplate(docs, values); + sorter.quickSort(0, docs.length - 1); + + final int[] oldToNew = new int[docs.length]; + for (int i = 0; i < docs.length; i++) { + oldToNew[docs[i]] = i; + } + return oldToNew; + } + + /** + * Returns a mapping from the old document ID to its new location in the + * sorted index. Implementations can use the auxiliary + * {@link #compute(int[], List)} to compute the old-to-new permutation + * given an array of documents and their corresponding values. + *

+ * NOTE: deleted documents are expected to appear in the mapping as + * well, they will however be dropped when the index is actually sorted. + */ + public abstract int[] oldToNew(AtomicReader reader) throws IOException; + +} Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/StoredFieldsSorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/StoredFieldsSorter.java (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/StoredFieldsSorter.java (copie de travail) @@ -0,0 +1,51 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.StoredDocument; + +/** + * A {@link Sorter} which sorts documents according to their stored fields. + * + * @lucene.experimental + */ +public abstract class StoredFieldsSorter> extends Sorter { + + /** + * Parse the given {@link StoredDocument} into a {@link Comparable} by which + * to sort the documents in the index. + */ + protected abstract T parse(StoredDocument doc); + + @Override + public final int[] oldToNew(final AtomicReader reader) throws IOException { + final int maxDoc = reader.maxDoc(); + int[] docs = new int[maxDoc]; + @SuppressWarnings({"unchecked"}) T[] values = (T[]) new Comparable[maxDoc]; + for (int i = 0; i < maxDoc; i++) { + docs[i] = i; + values[i] = parse(reader.document(i)); + } + return compute(docs, Arrays.asList(values)); + } + +} Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/StoredFieldsSorter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/PayloadSorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/PayloadSorter.java (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/PayloadSorter.java (copie de travail) @@ -0,0 +1,72 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BytesRef; + +/** + * A {@link Sorter} which sorts a document according to its payload. + * + * @lucene.experimental + */ +public abstract class PayloadSorter> extends Sorter { + + private final Term term; + + public PayloadSorter(Term term) { + this.term = term; + } + + /** + * Parse the given {@link BytesRef} into a {@link Comparable} by which + * to sort the documents in the index. + */ + protected abstract T parse(BytesRef payload); + + @Override + public final int[] oldToNew(final AtomicReader reader) throws IOException { + TermsEnum termsEnum = reader.fields().terms(term.field()).iterator(null); + final boolean found = termsEnum.seekExact(term.bytes(), false); + if (!found) { + throw new IllegalArgumentException(term + " not found in reader"); + } + DocsAndPositionsEnum it = termsEnum.docsAndPositions(null, null); + final int maxDoc = reader.maxDoc(); + int[] docs = new int[maxDoc]; + @SuppressWarnings({"unchecked"}) T[] values = (T[]) new Comparable[maxDoc]; + int i = 0; + int doc; + while ((doc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + assert i == doc : "all index documents must exist in the payload; missing " + i; + it.nextPosition(); + docs[i] = doc; + values[i] = parse(it.getPayload()); + i++; + } + return compute(docs, Arrays.asList(values)); + } + +} Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/PayloadSorter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (révision 1453428) +++ lucene/CHANGES.txt (copie de travail) @@ -195,6 +195,10 @@ * LUCENE-4748: Added DrillSideways utility class for computing both drill-down and drill-sideways counts for a DrillDownQuery. (Mike McCandless) + +* LUCENE-3918: IndexSorter has been ported to 4.0/trunk API and now supports + sorting documents by different sources of values, such as stored fields, + DocValues and payload. (Anat Hashavit, Shai Erera) API Changes