Index: lucene/misc/src/java/org/apache/lucene/index/sorter/Doc.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/Doc.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/Doc.java (working copy) @@ -0,0 +1,54 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.util.Arrays; + +/** + * This class represents a document that is to be sorted in the index. + * @param A comparable type according to which the document should be sorted. + * + * @lucene.experimental + */ + +public class Doc> implements Comparable> { + private final T comparable; + private final int id; + + /** Returns a permutation on the list of documents from their id's to + * their order when sorted according to T */ + public static int[] old2new(final Doc[] docs) { + Arrays.sort(docs); + final int[] oldToNew = new int[docs.length]; + for (int i = 0; i < docs.length; i++) + oldToNew[docs[i].id] = i; + return oldToNew; + } + + public Doc(final T comparable, final int docId) { + this.comparable = comparable; + this.id = docId; + } + + @Override + public int compareTo(final Doc doc) { + final int compareTo = comparable.compareTo(doc.comparable); + if (compareTo != 0) return compareTo; + return id - doc.id; + } + +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/DocValuesSorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/DocValuesSorter.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/DocValuesSorter.java (working copy) @@ -0,0 +1,56 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + +/** + * + * A {@link Sorter} which sorts documents according to their DocValues. + * + * @lucene.experimental + */ +public class DocValuesSorter> implements Sorter { + + private final To to; + private final String fieldName; + + public DocValuesSorter(final String fieldName, final To to) { + this.to = to; + this.fieldName = fieldName; + } + + @Override + public int[] oldToNew(final Directory dir) throws IOException { + final DirectoryReader reader = DirectoryReader.open(dir); + final AtomicReader atomicReader = new SlowCompositeReaderWrapper(reader); // TODO: consider optimizing later + final int maxDoc = reader.maxDoc(); + final Doc[] docs = new Doc[maxDoc]; + BytesRef bytes = new BytesRef(); + for (int i = 0; i < maxDoc; i++) { + bytes = atomicReader.docValues(fieldName).getSource().getBytes(i, bytes); + docs[i] = new Doc(to.from(bytes), i); + } + atomicReader.close(); + return Doc.old2new(docs); + } +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/DocumentSorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/DocumentSorter.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/DocumentSorter.java (working copy) @@ -0,0 +1,48 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.store.Directory; + +/** + * A {@link Sorter} which sorts documents according to their stored fields. + * + * @lucene.experimental + */ +public class DocumentSorter> implements Sorter { + + private final To to; + + public DocumentSorter(final To to) { + this.to = to; + } + + @Override + public int[] oldToNew(final Directory dir) throws IOException { + final DirectoryReader reader = DirectoryReader.open(dir); + final int maxDoc = reader.maxDoc(); + final Doc[] docs = new Doc[maxDoc]; + for (int i = 0; i < maxDoc; i++) + docs[i] = new Doc(to.from(reader.document(i)), i); + reader.close(); + return Doc.old2new(docs); + } + +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/PayloadSorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/PayloadSorter.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/PayloadSorter.java (working copy) @@ -0,0 +1,58 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + +/** + * A {@link Sorter} which sorts a document according to it's payload. + * + * @lucene.experimental + */ +public class PayloadSorter> implements Sorter { + + private final To to; + private final BytesRef term; + private final String field; + + public PayloadSorter(final BytesRef term, final String field, final To to) { + this.to = to; + this.field = field; + this.term = term; + } + + @Override + public int[] oldToNew(final Directory dir) throws IOException { + final DirectoryReader reader = DirectoryReader.open(dir); + final DocsAndPositionsEnum it = MultiFields.getTermPositionsEnum(reader, null, field, term); + final int maxDoc = reader.maxDoc(); + final Doc[] docs = new Doc[maxDoc]; + int i = 0; + while (it.nextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS) { + it.nextPosition(); + docs[i] = new Doc(to.from(it.getPayload()), i); + i++; + } + reader.close(); + return Doc.old2new(docs); + } +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/ReverseDocIdSorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/ReverseDocIdSorter.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/ReverseDocIdSorter.java (working copy) @@ -0,0 +1,41 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.store.Directory; + +/** + * A {@link Sorter} which sorts document in a reverse order to their doc id's + * + * @lucene.experimental + */ +public class ReverseDocIdSorter implements Sorter { + + @Override + public int[] oldToNew(final Directory dir) throws IOException { + final DirectoryReader reader = DirectoryReader.open(dir); + final int maxDoc = reader.maxDoc(); + int[] reverseDocs = new int[maxDoc]; + for (int i = 0; i < maxDoc; i++) + reverseDocs[i] = maxDoc - (i + 1); + reader.close(); + return reverseDocs; + } + +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (working copy) @@ -0,0 +1,31 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import org.apache.lucene.store.Directory; +/** + * An interface for sorting documents in a directory. + * + * @lucene.experimental + */ +public interface Sorter { + + /** Returns a list of document id's of the document in the received directory, + * sorted according to their new order. */ + int[] oldToNew(Directory dir) throws IOException; +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SorterUtil.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SorterUtil.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SorterUtil.java (working copy) @@ -0,0 +1,52 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Version; + +/*** + * A utility class for index sorting. + * + * @lucene.experimental + */ +public class SorterUtil { + + /** + * + * @param in A directory of documents to be sorted + * @param out The directory to which the sorted documents will be added + * @param sorter An object that knows how to sort the documents. + * @throws IOException + */ + public static void sort(Directory in, Directory out, Sorter sorter) throws IOException { + int[] old2new = sorter.oldToNew(in); + final IndexWriter writer = new IndexWriter(out, new IndexWriterConfig(Version.LUCENE_40, new StandardAnalyzer(Version.LUCENE_40))); + final DirectoryReader reader = DirectoryReader.open(in); + final SortingIndexReader sortingReader = new SortingIndexReader(reader, old2new); + for (int i = 0; i < old2new.length; i++) + writer.addDocument(sortingReader.document(i)); + sortingReader.close(); + writer.close(); + } + +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingDocValues.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingDocValues.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingDocValues.java (working copy) @@ -0,0 +1,65 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValues.Source; +import org.apache.lucene.index.DocValues.Type; + +/** + * Allow {@link DocValues} to be read according to old2new permutation. + * + * @lucene.experimental + */ +public class SortingDocValues extends DocValues { + + private final DocValues oldDocValues; + private final int[] old2new; + + public SortingDocValues(final DocValues oldDocValues, final int[] old2new) { + this.oldDocValues = oldDocValues; + this.old2new = old2new; + } + + @Override + public Source getDirectSource() throws IOException { + return new SortingSource(oldDocValues.getDirectSource(), old2new); + } + + /*** + * This operation is not supported. + */ + @Override + public Source getSource() throws IOException { + throw new UnsupportedOperationException("getSource is not supported"); + } + + @Override + public Type getType() { + return oldDocValues.getType(); + } + + /*** + * This operation is not supported. + */ + @Override + public Source load() throws IOException { + throw new UnsupportedOperationException("load is not supported"); + } + +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingDocsAndPositionsEnum.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingDocsAndPositionsEnum.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingDocsAndPositionsEnum.java (working copy) @@ -0,0 +1,190 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.RAMInputStream; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; +import sun.jdbc.odbc.OdbcDef; +/** + * + * Allow {@link DocsAndPositionsEnum} to be read according to old2new permutation. + * @lucene.experimental + */ +public class SortingDocsAndPositionsEnum extends DocsAndPositionsEnum { + + static class Position { + final int pos; + final int startOffset; + final int endOffset; + final BytesRef payload; + + public Position(final int pos, final int startOffset, final int endOffset, final BytesRef payload) { + this.pos = pos; + this.startOffset = startOffset; + this.endOffset = endOffset; + this.payload = payload; + } + } + + private static class PostingMap implements Comparable { + private final int newDocId; + private final long offset; + + PostingMap(final int newDocId, final long offset) { + this.newDocId = newDocId; + this.offset = offset; + } + @Override + public int compareTo(PostingMap pm) { // order by newDoc id + return this.newDocId - pm.newDocId; + } + } + + private static final int OFFSETS_INIT_LENGTH = 0; + private static final int OFFSETS_GROW_LENGTH = 1; + + private PostingMap[] postingMaps = new PostingMap[OFFSETS_INIT_LENGTH]; + private final int upto; + + private static final String TEMP_FILE = "temp"; + private final RAMDirectory tempDir = new RAMDirectory(); + private final IndexOutput out; + private final IndexInput in; + + int docIt = -1, posIt = -1; + private Position currentPosition; + private int currFreq; + + public SortingDocsAndPositionsEnum(final DocsAndPositionsEnum oldDocsAndPositions, final int[] old2new) throws IOException { + int i = 0; + out = tempDir.createOutput(TEMP_FILE, null); + + while (oldDocsAndPositions.nextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS) { + if ( i >= postingMaps.length) { + PostingMap[] newArray = new PostingMap[ArrayUtil.oversize( + postingMaps.length + OFFSETS_GROW_LENGTH, RamUsageEstimator.NUM_BYTES_SHORT)]; + System.arraycopy(postingMaps, 0, newArray, 0, postingMaps.length); + postingMaps = newArray; + } + postingMaps[i] = new PostingMap(old2new[oldDocsAndPositions.docID()],out.getFilePointer()); + addPositions(oldDocsAndPositions); + i++; + } + upto = i; + Arrays.sort(postingMaps, 0, upto); + + out.flush(); + in = tempDir.openInput(TEMP_FILE, IOContext.READ); + } + + /*** + * This operation is not supported. + */ + @Override + public int advance(final int target) throws IOException { + throw new UnsupportedOperationException("advance is not supported"); + } + + /*** + * This operation is not supported. + */ + @Override + public int docID() { + throw new UnsupportedOperationException("docID is not supported"); + } + + @Override + public int endOffset() throws IOException { + return currentPosition.endOffset; + } + + @Override + public int freq() throws IOException { + return currFreq; + } + + @Override + public BytesRef getPayload() throws IOException { + return currentPosition.payload; + } + + @Override + public int nextDoc() throws IOException { + posIt = -1; + if (++docIt >= upto) return DocsAndPositionsEnum.NO_MORE_DOCS; + in.seek(postingMaps[docIt].offset); + currFreq = in.readVInt(); + return postingMaps[docIt].newDocId; + } + + @Override + public int nextPosition() throws IOException { + final int pos = in.readVInt(); + final int startOffset = in.readVInt(); + final int endOffset = in.readVInt(); + final int offset = in.readVInt(); + final int length = in.readVInt(); + byte[] bytes = new byte[length]; + in.readBytes(bytes, offset,length); + final BytesRef payload = new BytesRef(bytes, offset, length); + currentPosition = new Position(pos, startOffset, endOffset, payload); + return currentPosition.pos; + } + + @Override + public int startOffset() throws IOException { + return currentPosition.startOffset; + } + + private void addPositions(final DocsAndPositionsEnum oldDocsAndPositions) throws IOException { + + out.writeVInt(oldDocsAndPositions.freq()); + for (int i = 0; i < oldDocsAndPositions.freq(); i++) { + final int oldPos = oldDocsAndPositions.nextPosition(); + + out.writeVInt(oldPos); + out.writeVInt(oldDocsAndPositions.startOffset()); + out.writeVInt(oldDocsAndPositions.endOffset()); + + BytesRef oldPayload = oldDocsAndPositions.getPayload(); + if (oldPayload != null) { + oldPayload = BytesRef.deepCopyOf(oldPayload); + out.writeVInt(oldPayload.offset); + out.writeVInt(oldPayload.length); + out.writeBytes(oldPayload.bytes, oldPayload.offset, oldPayload.length); + } else { + out.writeBytes(BytesRef.EMPTY_BYTES, 0, 0); + } + } + } +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingDocsEnum.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingDocsEnum.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingDocsEnum.java (working copy) @@ -0,0 +1,83 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.util.ArrayUtil; + +/** + * Allow {@link DocsEnum} to be read according to old2new permutation. + * @lucene.experimental + */ + + +public class SortingDocsEnum extends DocsEnum { + + private static final int NEW_DOC_ID_INIT_LENGTH = 0; + private static final int NEW_DOC_ID_GROW_LENGTH = 1; + + private int[] newDocIds = new int[NEW_DOC_ID_INIT_LENGTH]; + private int docIt = -1; + private final int upto; + + public SortingDocsEnum(final DocsEnum docs, final int[] old2new) throws IOException { + int i = 0; + while (docs.nextDoc() != DocsEnum.NO_MORE_DOCS){ + if ( i >= newDocIds.length) { + newDocIds = ArrayUtil.grow(newDocIds, newDocIds.length + NEW_DOC_ID_GROW_LENGTH); + } + newDocIds[i++] = old2new[docs.docID()]; + } + upto = i; + Arrays.sort(this.newDocIds, 0, upto); + } + + /*** + * This operation is not supported. + */ + @Override + public int advance(final int target) throws IOException { + throw new UnsupportedOperationException("advance is not supported"); + } + + /*** + * This operation is not supported. + */ + @Override + public int docID() { + throw new UnsupportedOperationException("docID is not supported"); + } + + /*** + * This operation is not supported. + */ + @Override + public int freq() throws IOException { + throw new UnsupportedOperationException("freq is not supported"); + } + + @Override + public int nextDoc() throws IOException { + if (++docIt >= upto) return NO_MORE_DOCS; + return newDocIds[docIt]; + } + +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingFields.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingFields.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingFields.java (working copy) @@ -0,0 +1,55 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Iterator; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.sorter.SortingTerms; + +/** + * + * Allow {@link Fields} to be read according to old2new permutation. + * @lucene.experimental + */ +public class SortingFields extends Fields { + + private final Fields oldFields; + private final int[] old2new; + + public SortingFields(final Fields oldFields, final int[] old2new) { + this.oldFields = oldFields; + this.old2new = old2new; + } + + @Override + public Iterator iterator() { + return oldFields.iterator(); + } + + @Override + public int size() { + return oldFields.size(); + } + + @Override + public Terms terms(final String field) throws IOException { + return new SortingTerms(oldFields.terms(field), old2new); + } + +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingIndexReader.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingIndexReader.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingIndexReader.java (working copy) @@ -0,0 +1,112 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.CompositeReader; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.util.Bits; + + +/** + * Allow {@link AtomicReader} to be read according to old2new permutation. + * Note that at this point deletion of document is not supported + * @lucene.experimental + */ +public class SortingIndexReader extends AtomicReader { + + private final AtomicReader oldReader; + private final int[] old2new; + + /** + * + * @param dirReader + * @param old2new + * @throws IOException + */ + public SortingIndexReader(final CompositeReader dirReader, final int[] old2new) throws IOException { + oldReader = new SlowCompositeReaderWrapper(dirReader); + this.old2new = old2new; + } + + @Override + public void document(final int docID, final StoredFieldVisitor visitor) throws IOException { + oldReader.document(old2new[docID], visitor); + } + + @Override + public DocValues docValues(final String field) throws IOException { + final DocValues oldDocValues = oldReader.docValues(field); + if (oldDocValues == null) return null; + return new SortingDocValues(oldDocValues, old2new); + } + + @Override + public Fields fields() throws IOException { + return new SortingFields(oldReader.fields(), old2new); + } + + @Override + public FieldInfos getFieldInfos() { + return oldReader.getFieldInfos(); + } + + @Override + public Bits getLiveDocs() { + return null; // default - no deleted docs + } + + @Override + public Fields getTermVectors(final int docID) throws IOException { + return oldReader.getTermVectors(old2new[docID]); + } + + /*** + * This operation is not supported. + */ + @Override + public boolean hasDeletions() { + throw new UnsupportedOperationException("hasDeletions is not supported"); + } + + @Override + public int maxDoc() { + return oldReader.maxDoc(); + } + + @Override + public DocValues normValues(final String field) throws IOException { + final DocValues oldDocValues = oldReader.normValues(field); + if (oldDocValues == null) return null; + return new SortingDocValues(oldDocValues, old2new); + } + + @Override + public int numDocs() { + return oldReader.numDocs(); + } + + @Override + protected void doClose() throws IOException { + oldReader.close(); + } +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingSource.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingSource.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingSource.java (working copy) @@ -0,0 +1,54 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.DocValues.Source; +import org.apache.lucene.util.BytesRef; + +/** + * + * Allow {@link Source} to be read according to old2new permutation. + * @lucene.experimental + */ +public class SortingSource extends Source { + + private final Source oldSource; + private final int[] old2new; + + public SortingSource(final Source oldSource, final int[] old2new) { + super(oldSource.getType()); + this.oldSource = oldSource; + this.old2new = old2new; + } + + @Override + public BytesRef getBytes(final int docID, final BytesRef ref) { + return oldSource.getBytes(old2new[docID], ref); + } + + @Override + public double getFloat(final int docID) { + return oldSource.getFloat(old2new[docID]); + } + + @Override + public long getInt(final int docID) { + return oldSource.getInt(old2new[docID]); + } + +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingTermEnum.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingTermEnum.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingTermEnum.java (working copy) @@ -0,0 +1,99 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Comparator; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.index.sorter.SortingDocsAndPositionsEnum; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; + +/** + * + * Allow {@link TermsEnum} to be read according to old2new permutation. + * @lucene.experimental + */ +public class SortingTermEnum extends TermsEnum { + + private final TermsEnum oldIterator; + private final int[] old2new; + + public SortingTermEnum(final TermsEnum oldIterator, final int[] old2new) { + this.oldIterator = oldIterator; + this.old2new = old2new; + } + + @Override + public int docFreq() throws IOException { + return oldIterator.docFreq(); + } + + @Override + public DocsEnum docs(final Bits liveDocs, final DocsEnum reuse, final int flags) throws IOException { + return new SortingDocsEnum(oldIterator.docs(liveDocs, reuse, flags), old2new); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(final Bits liveDocs, final DocsAndPositionsEnum reuse, final int flags) throws IOException { + return new SortingDocsAndPositionsEnum(oldIterator.docsAndPositions(liveDocs, reuse, flags), old2new); + } + + @Override + public Comparator getComparator() { + return oldIterator.getComparator(); + } + + @Override + public BytesRef next() throws IOException { + return oldIterator.next(); + } + + /*** + * This operation is not supported. + */ + @Override + public long ord() throws IOException { + throw new UnsupportedOperationException("ord is not supported"); + } + + @Override + public SeekStatus seekCeil(final BytesRef text, final boolean useCache) throws IOException { + return oldIterator.seekCeil(text, useCache); + } + + /*** + * This operation is not supported. + */ + @Override + public void seekExact(final long ord) throws IOException { + throw new UnsupportedOperationException("seekExact is not supported"); + } + + @Override + public BytesRef term() throws IOException { + return oldIterator.term(); + } + + @Override + public long totalTermFreq() throws IOException { + return oldIterator.totalTermFreq(); + } +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingTerms.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingTerms.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingTerms.java (working copy) @@ -0,0 +1,84 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Comparator; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; + +/** + * + * Allow {@link Terms} to be read according to old2new permutation. + * @lucene.experimental + */ +public class SortingTerms extends Terms { + + private final Terms oldTerms; + private final int[] old2new; + + public SortingTerms(final Terms oldTerms, final int[] old2new) { + this.oldTerms = oldTerms; + this.old2new = old2new; + } + + @Override + public Comparator getComparator() throws IOException { + return oldTerms.getComparator(); + } + + @Override + public int getDocCount() throws IOException { + return oldTerms.getDocCount(); + } + + @Override + public long getSumDocFreq() throws IOException { + return oldTerms.getSumDocFreq(); + } + + @Override + public long getSumTotalTermFreq() throws IOException { + return oldTerms.getSumTotalTermFreq(); + } + + @Override + public boolean hasOffsets() { + return oldTerms.hasOffsets(); + } + + @Override + public boolean hasPayloads() { + return oldTerms.hasPayloads(); + } + + @Override + public boolean hasPositions() { + return oldTerms.hasPositions(); + } + + @Override + public TermsEnum iterator(final TermsEnum reuse) throws IOException { + return new SortingTermEnum(oldTerms.iterator(reuse), old2new); + } + + @Override + public long size() throws IOException { + return oldTerms.size(); + } +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/To.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/To.java (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/To.java (working copy) @@ -0,0 +1,32 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** + * + * Interface used by index {@link Sorter} + * + * @param Object to be sorted. + * @param Field according to which the sorting will be done. + * + * @lucene.experimental + */ +public interface To { + /** Retrieves from S the field according to which the sorting will be done*/ + public T from(S s); +} Index: lucene/misc/src/java/org/apache/lucene/index/sorter/package.html =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/package.html (revision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/package.html (working copy) @@ -0,0 +1,91 @@ + + + + + + + +Code to sort indices. +

Table Of Contents

+

+

    +
  1. Sorters API
    2. +
  2. Sorting classes
    3. +
+

+ +

Sorters API

+

+All sorters implement the {@link org.apache.lucene.index.sorter.Sorter} interface. +Each sorter implementation sorts the documents in a directory according to the values of a document component such as +StoredFields, DocValues, Payload etc`. The relevant component is evident from the name of the sorter class. +Most sorter classes are generic and take in a parameter T which defines the type of the component +according to which the sorting should be done. This type must be comparable. +For example we can have an integer DocValues sorter, a double DocValues sorters and so on. +In order to use a generic sorter the user must supply the sorter with a {@link org.apache.lucene.index.sorter.To} object. +that can extract an object of a comparable type T according to which the sorting will be done. + +For example, sorting a directory according to an integer DocValues field will be done as follow: +

+  int[] docValuesSort(Directory dir) throws IOException{
+    final DocValuesSorter<Integer> docValSorter = new DocValuesSorter<Integer>("someField", new To<BytesRef, Integer>() {
+
+      @Override
+      public Integer from(final BytesRef bytes) {
+        return DocValuesArraySource.asInt(bytes);
+      }
+    });
+    return docValSorter.oldToNew(dir);    
+  }
+
+Currently the following sorters are implemented: +
    +
  1. {@link org.apache.lucene.index.sorter.DocumentSorter}
    2. +
  2. {@link org.apache.lucene.index.sorter.DocValuesSorter}
    3. +
  3. {@link org.apache.lucene.index.sorter.PayloadSorter}
    4. +
  4. {@link org.apache.lucene.index.sorter.ReverseDocIdSorter}
    5. +
+In addition, once a sorter is defined, the utility class {@link org.apache.lucene.index.sorter.SorterUtil} supplies +a convenient interface to read documents from an input directory, sort them and add them to an output +directory according to their new order. +

+ +

Sorting classes

+

+A sorting class is a class that extends a previously existing index class +and allows the index to be accessed according to a permutation on its original order. +Each sorting class received in its constructor an instance of its superclass +and a permutation +that defines the new order of the index. A sorting class exposes the same interface as its superclass +although not all of the superclasses functionality is currently supported. +
+The following sorting classes are available: +

    +
  1. {@link org.apache.lucene.index.sorter.SortingDocsAndPositionsEnum}
    2. +
  2. {@link org.apache.lucene.index.sorter.SortingDocsEnum}
    3. +
  3. {@link org.apache.lucene.index.sorter.SortingDocValues}
    4. +
  4. {@link org.apache.lucene.index.sorter.SortingFields}
    5. +
  5. {@link org.apache.lucene.index.sorter.SortingIndexReader}
    6. +
  6. {@link org.apache.lucene.index.sorter.SortingSource}
    7. +
  7. {@link org.apache.lucene.index.sorter.SortingTermEnum}
    8. +
  8. {@link org.apache.lucene.index.sorter.SortingTerms}
    9. +
+

+ + + Index: lucene/misc/src/test/org/apache/lucene/index/sorter/DocValuesSorterTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/DocValuesSorterTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/DocValuesSorterTest.java (working copy) @@ -0,0 +1,70 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.codecs.DocValuesArraySource; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.IntDocValuesField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class DocValuesSorterTest extends LuceneTestCase{ + private static final String DOC_VAL = "docVal"; + private RAMDirectory dir; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + dir = new RAMDirectory(); + final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random()))); + writer.addDocument(doc(3)); + writer.addDocument(doc(2)); + writer.addDocument(doc(1)); + writer.close(); + } + + @Test + public void test() throws IOException { + final DocValuesSorter docValSorter = new DocValuesSorter(DOC_VAL, new To() { + + @Override + public Integer from(final BytesRef bytes) { + return DocValuesArraySource.asInt(bytes); + } + }); + + assertArrayEquals(new int[] { 2, 1, 0 }, docValSorter.oldToNew(dir)); + } + + private Iterable doc(final int val) { + final Document doc = new Document(); + doc.add(new IntDocValuesField(DOC_VAL, val)); + return doc; + } + +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/DocumentSorterTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/DocumentSorterTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/DocumentSorterTest.java (working copy) @@ -0,0 +1,71 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.IntField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class DocumentSorterTest extends LuceneTestCase { + + private static final String VAL = "val"; + private RAMDirectory dir; + + @Override +@Before + public void setUp() throws Exception { + super.setUp(); + dir = new RAMDirectory(); + final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random()))); + writer.addDocument(doc(1)); + writer.addDocument(doc(3)); + writer.addDocument(doc(2)); + writer.addDocument(doc(1)); + writer.close(); + } + + @Test + public void test() throws IOException { + final DocumentSorter documentSorter = new DocumentSorter(new To() { + + @Override + public Integer from(final Document doc) { + return doc.getField(VAL).numericValue().intValue(); + } + }); + + assertArrayEquals(new int[] { 0, 3, 2, 1 }, documentSorter.oldToNew(dir)); + } + + private Iterable doc(final int val) { + final Document doc = new Document(); + doc.add(new IntField(VAL, val, Store.YES)); + return doc; + } + +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/PayloadSorterTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/PayloadSorterTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/PayloadSorterTest.java (working copy) @@ -0,0 +1,112 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.payloads.PayloadHelper; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class PayloadSorterTest extends LuceneTestCase{ + private static final String PAYLOAD_FIELD = "payloadField"; + private static final String TERM = "term"; + private RAMDirectory dir; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + dir = new RAMDirectory(); + final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random()))); + writer.addDocument(doc(3)); + writer.addDocument(doc(2)); + writer.addDocument(doc(1)); + writer.close(); + } + + @Test + public void test() throws IOException { + Term term = new Term(PAYLOAD_FIELD, TERM); + final PayloadSorter payloadSorter = new PayloadSorter(term.bytes(), term.field(), new To() { + + @Override + public Integer from(final BytesRef bytes) { + return PayloadHelper.decodeInt(bytes.bytes, bytes.offset); + + } + }); + + assertArrayEquals(new int[] { 2, 1, 0 }, payloadSorter.oldToNew(dir)); + } + + // TODO write a util that grab all the information for a particular document (terms, positions, payloads, etc....) + // TODO sort the index and then sort it back, use Lucen's util to compare the original index to the result one. + private void addPayload(final Document doc, final Integer id) { + final FieldType fieldType = new FieldType(); + fieldType.setIndexed(true); + fieldType.freeze(); + + doc.add(new Field(PAYLOAD_FIELD, new TokenStream() { + int positionsCount = 2; + private final CharTermAttribute attCharTerm = addAttribute(CharTermAttribute.class); + private final PayloadAttribute attPayload = addAttribute(PayloadAttribute.class); + private final OffsetAttribute attOffset = addAttribute(OffsetAttribute.class); + + @Override + public boolean incrementToken() throws IOException { + if (positionsCount-- == 0) return false; + attCharTerm.setEmpty().append(TERM); + attPayload.setPayload(new BytesRef(PayloadHelper.encodeInt(id))); + attOffset.setOffset(id, id); + return true; + } + + }, fieldType)); + } + + private Iterable doc(final Integer val) { + final Document doc = new Document(); + + doc.add(new StringField("stringFields", val.toString(), Store.YES)); + doc.add(new IntField("intField", val, Store.YES)); + addPayload(doc, val); + + return doc; + } + +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/ReverseDocIdSorterTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/ReverseDocIdSorterTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/ReverseDocIdSorterTest.java (working copy) @@ -0,0 +1,63 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.IntField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class ReverseDocIdSorterTest extends LuceneTestCase{ + + private static final String VAL = "val"; + private RAMDirectory dir; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + dir = new RAMDirectory(); + final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random()))); + writer.addDocument(doc(3)); + writer.addDocument(doc(2)); + writer.addDocument(doc(1)); + writer.close(); + } + + @Test + public void test() throws IOException { + final ReverseDocIdSorter documentSorter = new ReverseDocIdSorter(); + assertArrayEquals(new int[] { 2, 1, 0 }, documentSorter.oldToNew(dir)); + } + + private Iterable doc(final int val) { + final Document doc = new Document(); + doc.add(new IntField(VAL, val, Store.YES)); + return doc; + } + +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SorterUtilTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SorterUtilTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SorterUtilTest.java (working copy) @@ -0,0 +1,78 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.IntField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class SorterUtilTest extends LuceneTestCase{ + + private static final String VAL = "val"; + private RAMDirectory in; + private RAMDirectory out; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + in = new RAMDirectory(); + final IndexWriter writer = new IndexWriter(in, new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random()))); + writer.addDocument(doc(1)); + writer.addDocument(doc(3)); + writer.addDocument(doc(2)); + writer.close(); + out = new RAMDirectory(); + } + + @Test + public void test() throws IOException { + final DocumentSorter documentSorter = new DocumentSorter(new To() { + + @Override + public Integer from(final Document doc) { + return doc.getField(VAL).numericValue().intValue(); + } + }); + SorterUtil.sort(in, out, documentSorter); + final int[] expected = { 1, 2, 3 }; + final DirectoryReader outReader = DirectoryReader.open(out); + for (int i = 0; i < outReader.maxDoc(); i++) { + assertEquals(expected[i], outReader.document(i).getField(VAL).numericValue().intValue()); + } + + } + + private Iterable doc(final int val) { + final Document doc = new Document(); + doc.add(new IntField(VAL, val, Store.YES)); + return doc; + } + +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SortingDocValuesTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SortingDocValuesTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SortingDocValuesTest.java (working copy) @@ -0,0 +1,98 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.IntDocValuesField; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValues.Source; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class SortingDocValuesTest extends LuceneTestCase{ + + AtomicReader atomicReader; + + static final IndexWriterConfig INDEX_WRITER_CONFIG = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random())); + private final int[] oldToNew = { 2, 1, 0 }; + + private static final String VAL = "Val"; + private SortingDocValues sortingDocValues; + private DocValues unsortedDocValues; + + private Document doc(final Integer id) { + final Document doc = new Document(); + + doc.add(new IntDocValuesField(VAL, id)); + return doc; + + } + +@Override +@Before + public void setUp() throws Exception { + super.setUp(); + Directory dir = new RAMDirectory(); + final IndexWriter indexWriter = new IndexWriter(dir, INDEX_WRITER_CONFIG); + + indexWriter.addDocument(doc(0)); + indexWriter.addDocument(doc(1)); + indexWriter.addDocument(doc(2)); + indexWriter.close(); + + final DirectoryReader dirReader = DirectoryReader.open(dir); + atomicReader = new SlowCompositeReaderWrapper(dirReader); + unsortedDocValues = atomicReader.docValues(VAL); + sortingDocValues = new SortingDocValues(unsortedDocValues, oldToNew); + } + + @Test + public void testGetDirectSource() { + try { + Source unsortedSource = unsortedDocValues.getDirectSource(); + Source sortedSource = sortingDocValues.getDirectSource(); + + assertEquals(unsortedSource.getInt(2), sortedSource.getInt(0)); + assertEquals(unsortedSource.getInt(1), sortedSource.getInt(1)); + assertEquals(unsortedSource.getInt(0), sortedSource.getInt(2)); + + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + fail("Exception"); + } + } + + + @Test + public void testGetType() { + assertEquals(unsortedDocValues.getType(), sortingDocValues.getType()); + } + +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SortingDocsAndPositionsEnumTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SortingDocsAndPositionsEnumTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SortingDocsAndPositionsEnumTest.java (working copy) @@ -0,0 +1,268 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class SortingDocsAndPositionsEnumTest extends LuceneTestCase{ + AtomicReader atomicReader; + DirectoryReader dirReader; + + static final IndexWriterConfig INDEX_WRITER_CONFIG = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random())); + private final int[] oldToNew = { 2, 1, 0 }; + + private static final String PAYLOAD_TEST_CONTENT = "payload"; + + private SortingDocsAndPositionsEnum sortingEnum; + private DocsAndPositionsEnum unSortedEnum; + + private void addPayload(final Document doc, final Integer id) { + final FieldType fieldType = new FieldType(); + fieldType.setIndexed(true); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + fieldType.freeze(); + + doc.add(new Field(PAYLOAD_TEST_CONTENT, new TokenStream() { + int positionsCount = id+1; + private final CharTermAttribute attCharTerm = addAttribute(CharTermAttribute.class); + private final PayloadAttribute attPayload = addAttribute(PayloadAttribute.class); + private final OffsetAttribute attOffset = addAttribute(OffsetAttribute.class); + + @Override + public boolean incrementToken() throws IOException { + if (positionsCount-- == 0) return false; + + attCharTerm.setEmpty().append(PAYLOAD_TEST_CONTENT); + attPayload.setPayload(new BytesRef(id.toString())); + attOffset.setOffset(id, id); + return true; + } + + }, fieldType)); + } + + private Document doc(final Integer id) { + final Document doc = new Document(); + addPayload(doc, id); + return doc; + } + +@Override +@Before + public void setUp() throws Exception { + super.setUp(); + Directory dir = new RAMDirectory(); + final IndexWriter indexWriter = new IndexWriter(dir, INDEX_WRITER_CONFIG); + + indexWriter.addDocument(doc(0)); + indexWriter.addDocument(doc(1)); + indexWriter.addDocument(doc(2)); + indexWriter.close(); + + dirReader = DirectoryReader.open(dir); + atomicReader = new SlowCompositeReaderWrapper(dirReader); + unSortedEnum = MultiFields.getTermPositionsEnum(dirReader, + null, PAYLOAD_TEST_CONTENT, new BytesRef(PAYLOAD_TEST_CONTENT)); + + DocsAndPositionsEnum oldEnum = MultiFields + .getTermPositionsEnum(dirReader, null, PAYLOAD_TEST_CONTENT, new BytesRef(PAYLOAD_TEST_CONTENT)); + sortingEnum = new SortingDocsAndPositionsEnum(oldEnum, oldToNew); + + } + + @Test + public void testNextDoc() { + try { + assertEquals(sortingEnum.nextDoc(), 0); + assertEquals(sortingEnum.nextDoc(), 1); + assertEquals(sortingEnum.nextDoc(), 2); + assertEquals(sortingEnum.nextDoc(), DocsAndPositionsEnum.NO_MORE_DOCS); + } catch (IOException e) { + fail("Exception"); + } + + } + + @Test + public void testFreq() { + try { + assertEquals(sortingEnum.nextDoc(), 0); + assertEquals(sortingEnum.freq(), 3); + assertEquals(sortingEnum.nextDoc(), 1); + assertEquals(sortingEnum.freq(), 2); + assertEquals(sortingEnum.nextDoc(), 2); + assertEquals(sortingEnum.freq(), 1); + } catch (IOException e) { + fail("Exception"); + } + } + + @Test + public void testNextPosition() { + try { + int[][] unsortedPositions = new int[3][]; + int[][] sortedPositions = new int[3][]; + for ( int i=0; i < 3; i++) { + unSortedEnum.nextDoc(); + unsortedPositions[i] = new int[unSortedEnum.freq()]; + for (int j = 0; j < unsortedPositions[i].length; j++) { + unsortedPositions[i][j] = unSortedEnum.nextPosition(); + } + + sortingEnum.nextDoc(); + sortedPositions[i] = new int[sortingEnum.freq()]; + for (int j = 0; j < sortedPositions[i].length; j++) { + sortedPositions[i][j] = sortingEnum.nextPosition(); + } + } + assertEquals(unSortedEnum.nextDoc(), DocsAndPositionsEnum.NO_MORE_DOCS); + assertEquals(sortingEnum.nextDoc(), DocsAndPositionsEnum.NO_MORE_DOCS); + + for (int i=0; i < sortedPositions.length; i++) { + assertArrayEquals(sortedPositions[i], unsortedPositions[2-i]); + } + } catch (IOException e) { + fail("Exception"); + } + } + + @Test + public void testStartOffset() { + try { + int[][] unsortedOffsets = new int[3][]; + int[][] sortedOffsets = new int[3][]; + + for ( int i=0; i < 3; i++) { + unSortedEnum.nextDoc(); + unsortedOffsets[i] = new int[unSortedEnum.freq()]; + for (int j = 0; j < unsortedOffsets[i].length; j++) { + unSortedEnum.nextPosition(); + unsortedOffsets[i][j] = unSortedEnum.startOffset(); + } + + sortingEnum.nextDoc(); + sortedOffsets[i] = new int[sortingEnum.freq()]; + for (int j = 0; j < sortedOffsets[i].length; j++) { + sortingEnum.nextPosition(); + sortedOffsets[i][j] = sortingEnum.startOffset(); + } + } + assertEquals(unSortedEnum.nextDoc(), DocsAndPositionsEnum.NO_MORE_DOCS); + assertEquals(sortingEnum.nextDoc(), DocsAndPositionsEnum.NO_MORE_DOCS); + + for (int i=0; i < sortedOffsets.length; i++) { + assertArrayEquals(sortedOffsets[i], unsortedOffsets[2-i]); + } + + } catch (IOException e) { + fail("Exception"); + } + } + + @Test + public void testEndOffset() { + try { + int[][] unsortedOffsets = new int[3][]; + int[][] sortedOffsets = new int[3][]; + + for ( int i=0; i < 3; i++) { + unSortedEnum.nextDoc(); + unsortedOffsets[i] = new int[unSortedEnum.freq()]; + for (int j = 0; j < unsortedOffsets[i].length; j++) { + unSortedEnum.nextPosition(); + unsortedOffsets[i][j] = unSortedEnum.endOffset(); + } + + sortingEnum.nextDoc(); + sortedOffsets[i] = new int[sortingEnum.freq()]; + for (int j = 0; j < sortedOffsets[i].length; j++) { + sortingEnum.nextPosition(); + sortedOffsets[i][j] = sortingEnum.endOffset(); + } + } + assertEquals(unSortedEnum.nextDoc(), DocsAndPositionsEnum.NO_MORE_DOCS); + assertEquals(sortingEnum.nextDoc(), DocsAndPositionsEnum.NO_MORE_DOCS); + + for (int i=0; i < sortedOffsets.length; i++) { + assertArrayEquals(sortedOffsets[i], unsortedOffsets[2-i]); + } + + } catch (IOException e) { + fail("Exception"); + } + + } + + @Test + public void testGetPayload() { + try { + BytesRef[][] unsortedPayloads = new BytesRef[3][]; + BytesRef[][] sortedPayloads = new BytesRef[3][]; + + for ( int i=0; i < 3; i++) { + unSortedEnum.nextDoc(); + unsortedPayloads[i] = new BytesRef[unSortedEnum.freq()]; + for (int j = 0; j < unsortedPayloads[i].length; j++) { + unSortedEnum.nextPosition(); + unsortedPayloads[i][j] = BytesRef.deepCopyOf(unSortedEnum.getPayload()); + } + + sortingEnum.nextDoc(); + sortedPayloads[i] = new BytesRef[sortingEnum.freq()]; + for (int j = 0; j < sortedPayloads[i].length; j++) { + sortingEnum.nextPosition(); + sortedPayloads[i][j] = BytesRef.deepCopyOf(sortingEnum.getPayload()); + } + } + assertEquals(unSortedEnum.nextDoc(), DocsAndPositionsEnum.NO_MORE_DOCS); + assertEquals(sortingEnum.nextDoc(), DocsAndPositionsEnum.NO_MORE_DOCS); + + for (int i=0; i < sortedPayloads.length; i++) { + assertArrayEquals(sortedPayloads[i], unsortedPayloads[2-i]); + } + + } catch (IOException e) { + fail("Exception"); + } + + + } +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SortingDocsEnumTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SortingDocsEnumTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SortingDocsEnumTest.java (working copy) @@ -0,0 +1,118 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class SortingDocsEnumTest extends LuceneTestCase{ + + private SortingDocsEnum sortingDocsEnum; + + static final IndexWriterConfig INDEX_WRITER_CONFIG = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random())); + private final int[] oldToNew = { 2, 1, 0 }; + + private static final String PAYLOAD_TEST_CONTENT = "payload"; + private static final String VAL = "Val"; + + private void addPayload(final Document doc, final Integer id) { + final FieldType fieldType = new FieldType(); + fieldType.setIndexed(true); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + fieldType.freeze(); + + doc.add(new Field(PAYLOAD_TEST_CONTENT, new TokenStream() { + int positionsCount = id+1; + private final CharTermAttribute attCharTerm = addAttribute(CharTermAttribute.class); + private final PayloadAttribute attPayload = addAttribute(PayloadAttribute.class); + private final OffsetAttribute attOffset = addAttribute(OffsetAttribute.class); + + @Override + public boolean incrementToken() throws IOException { + if (positionsCount-- == 0) return false; + + attCharTerm.setEmpty().append(PAYLOAD_TEST_CONTENT); + attPayload.setPayload(new BytesRef(id.toString())); + attOffset.setOffset(id, id); + return true; + } + + }, fieldType)); + } + + private Document doc(final Integer id) { + final Document doc = new Document(); + + doc.add(new IntField(VAL, id, Store.YES)); + addPayload(doc, id); + return doc; + } + +@Override +@Before + public void setUp() throws Exception { + super.setUp(); + Directory dir = new RAMDirectory(); + final IndexWriter indexWriter = new IndexWriter(dir, INDEX_WRITER_CONFIG); + + indexWriter.addDocument(doc(0)); + indexWriter.addDocument(doc(1)); + indexWriter.addDocument(doc(2)); + indexWriter.close(); + + final DirectoryReader dirReader = DirectoryReader.open(dir); + DocsEnum docsEnum = MultiFields.getTermDocsEnum(dirReader, null, PAYLOAD_TEST_CONTENT, new BytesRef(PAYLOAD_TEST_CONTENT)); + sortingDocsEnum = new SortingDocsEnum(docsEnum, oldToNew); + } + + + @Test + public void testNextDoc() { + try { + assertEquals(0, sortingDocsEnum.nextDoc()); + assertEquals(1, sortingDocsEnum.nextDoc()); + assertEquals(2, sortingDocsEnum.nextDoc()); + } catch (IOException e) { + fail("Exception"); + } + + } + +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SortingFieldsTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SortingFieldsTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SortingFieldsTest.java (working copy) @@ -0,0 +1,101 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Iterator; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.IntField; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class SortingFieldsTest extends LuceneTestCase{ + + AtomicReader atomicReader; + + static final IndexWriterConfig INDEX_WRITER_CONFIG = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random())); + private final int[] oldToNew = { 2, 1, 0 }; + + private static final String VAL = "Val"; + private SortingFields sortingFields; + + private Fields unsortedFields; + + private Document doc(final Integer id) { + final Document doc = new Document(); + doc.add(new IntField(VAL, id, Store.YES)); + return doc; + } + + @Override +@Before + public void setUp() throws Exception { + super.setUp(); + Directory dir = new RAMDirectory(); + final IndexWriter indexWriter = new IndexWriter(dir, INDEX_WRITER_CONFIG); + + indexWriter.addDocument(doc(0)); + indexWriter.addDocument(doc(1)); + indexWriter.addDocument(doc(2)); + indexWriter.close(); + + final DirectoryReader dirReader = DirectoryReader.open(dir); + atomicReader = new SlowCompositeReaderWrapper(dirReader); + unsortedFields = atomicReader.fields(); + sortingFields = new SortingFields(unsortedFields, oldToNew); + + } + + @Test + public void testSize() { + assertEquals(unsortedFields.size(), sortingFields.size()); + } + + @Test + public void testIterator() { + Iterator i1 = unsortedFields.iterator(); + Iterator i2 = sortingFields.iterator(); + while (i1.hasNext()) { + assertEquals(i1.next(), i2.next()); + } + assertFalse(i1.hasNext()); + assertFalse(i2.hasNext()); + } + + @Test + public void testTermsString() { + try { + TestUtils.compareTerms(unsortedFields.terms(VAL), sortingFields.terms(VAL)); + } catch (IOException e) { + fail("Exception"); + } + + } +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SortingIndexReaderTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SortingIndexReaderTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SortingIndexReaderTest.java (working copy) @@ -0,0 +1,286 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DocumentStoredFieldVisitor; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.FieldType.NumericType; +import org.apache.lucene.document.IntDocValuesField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.CompositeReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.ReaderClosedListener; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class SortingIndexReaderTest extends LuceneTestCase{ + + AtomicReader atomicReader; + + static final IndexWriterConfig INDEX_WRITER_CONFIG = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random())); + private final int[] oldToNew = { 2, 1, 0 }; + + private static final String VAL = "val"; + private static final String DOC_VAL = "docVal"; + + private SortingIndexReader sortingIndexReader; + private CompositeReader unsortedIndexReader; + private CompositeReader oldIndexReader; + + private Document doc(final Integer id) { + final Document doc = new Document(); + + FieldType ft = new FieldType(); + + doc.add(new IntDocValuesField(DOC_VAL, id)); + + // for the norm values test. We want each doc to have a different norm value + // therefore we add a different number of terms to each doc. + for (int i = 0; i <= id; i++) { + ft = new FieldType(); + + ft.setIndexed(true); + ft.setStored(true); + ft.setStoreTermVectors(true); + ft.setOmitNorms(false); + ft.setNumericType(NumericType.INT); + + doc.add(new IntField(VAL, id, ft)); + } + + return doc; + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + Directory dir = new RAMDirectory(); + INDEX_WRITER_CONFIG.setSimilarity(new DefaultSimilarity()); + final IndexWriter indexWriter = new IndexWriter(dir, INDEX_WRITER_CONFIG); + + indexWriter.addDocument(doc(0)); + indexWriter.addDocument(doc(1)); + indexWriter.addDocument(doc(2)); + indexWriter.close(); + + final DirectoryReader dirReader = DirectoryReader.open(dir); + atomicReader = new SlowCompositeReaderWrapper(dirReader); + unsortedIndexReader = DirectoryReader.open(dir); + oldIndexReader = DirectoryReader.open(dir); + sortingIndexReader = new SortingIndexReader(oldIndexReader, oldToNew); + } + + @Test + public void testNumDocs() { + assertEquals(unsortedIndexReader.numDocs(), sortingIndexReader.numDocs()); + } + + @Test + public void testMaxDoc() { + assertEquals(unsortedIndexReader.maxDoc(), sortingIndexReader.maxDoc()); + } + + + @Test + public void testDoClose() { + + class MyReaderClosedListener implements ReaderClosedListener { + + boolean closed; + + @Override + public void onClose(IndexReader arg0) { + closed = true; + + } + + } + + MyReaderClosedListener closedListener = new MyReaderClosedListener(); + + try { + oldIndexReader.addReaderClosedListener(closedListener); + sortingIndexReader.doClose(); + // wait for the listener + for (int i = 0; i < 1000; i++) { + if (closedListener.closed) { + assertTrue(true); + return; + } + Thread.sleep(1000); + } + fail("Reader not closed"); + } catch (IOException e) { + fail("Exception"); + } catch (InterruptedException e) { + fail("Exception"); + } + fail("Reader Not Closed"); + + } + + @Test + public void testDocumentIntStoredFieldVisitor() { + DocumentStoredFieldVisitor sortedVisitor = new DocumentStoredFieldVisitor(); + DocumentStoredFieldVisitor oldVisitor = new DocumentStoredFieldVisitor(); + try { + unsortedIndexReader.document(0, oldVisitor); + sortingIndexReader.document(2, sortedVisitor); + assertEquals(oldVisitor.getDocument().toString(), sortedVisitor.getDocument().toString()); + + unsortedIndexReader.document(1, oldVisitor); + sortingIndexReader.document(1, sortedVisitor); + assertEquals(oldVisitor.getDocument().toString(), sortedVisitor.getDocument().toString()); + + unsortedIndexReader.document(2, oldVisitor); + sortingIndexReader.document(0, sortedVisitor); + assertEquals(oldVisitor.getDocument().toString(), sortedVisitor.getDocument().toString()); + + } catch (IOException e) { + fail("Exception"); + } + } + + @Test + public void testDocValues() { + try { + assertNull(sortingIndexReader.docValues("")); + final DocValues docValues = sortingIndexReader.docValues(DOC_VAL); + assertEquals(2, docValues.getDirectSource().getInt(0)); + assertEquals(1, docValues.getDirectSource().getInt(1)); + assertEquals(0, docValues.getDirectSource().getInt(2)); + } catch (IOException e) { + fail("Exception"); + } + + } + + @Test + public void testFields() { + Fields sortedFields; + try { + sortedFields = sortingIndexReader.fields(); + Terms sortedTerms = sortedFields.terms(VAL); + Terms oldTerms = atomicReader.fields().terms(VAL); + TermsEnum sortedTermsIt = sortedTerms.iterator(null); + TermsEnum oldTermsIt = oldTerms.iterator(null); + assertEquals(oldTerms.size(), sortedTerms.size()); + for (int i = 0; i < oldTerms.size(); i++) { + assertEquals(oldTermsIt.next(), sortedTermsIt.next()); + } + + } catch (IOException e) { + fail("Exception"); + } + + } + + @Test + public void testGetLiveDocs() { + assertNull(sortingIndexReader.getLiveDocs()); + } + + @Test + public void testGetFieldInfos() { + FieldInfos oldReaderInfo = atomicReader.getFieldInfos(); + FieldInfos sortingReaderInfo = sortingIndexReader.getFieldInfos(); + Iterator oldInfoIt = oldReaderInfo.iterator(); + Iterator sortingInfoIt = sortingReaderInfo.iterator(); + while (sortingInfoIt.hasNext()) { + FieldInfo oldF = oldInfoIt.next(); + FieldInfo sortedF = sortingInfoIt.next(); + assertEquals(oldF.name, sortedF.name); + assertEquals(oldF.number, sortedF.number); + assertEquals(oldF.hasDocValues(), sortedF.hasDocValues()); + assertEquals(oldF.hasNorms(), sortedF.hasNorms()); + assertEquals(oldF.hasPayloads(), sortedF.hasPayloads()); + assertEquals(oldF.hasVectors(), sortedF.hasVectors()); + assertEquals(oldF.isIndexed(), sortedF.isIndexed()); + assertEquals(oldF.omitsNorms(), sortedF.omitsNorms()); + assertEquals(oldF.getDocValuesType(), sortedF.getDocValuesType()); + assertEquals(oldF.getIndexOptions(), sortedF.getIndexOptions()); + assertEquals(oldF.getNormType(), sortedF.getNormType()); + if (sortedF.attributes() == null) { + assertNull(oldF.attributes()); + } else { + Iterator> sortedMap = sortedF.attributes().entrySet().iterator(); + Iterator> oldMap = oldF.attributes().entrySet().iterator(); + + while (sortedMap.hasNext()) { + Entry sortedEntry = sortedMap.next(); + Entry oldEntry = oldMap.next(); + assertEquals(sortedEntry.getKey(), oldEntry.getKey()); + assertEquals(sortedEntry.getValue(), oldEntry.getValue()); + } + assertFalse(oldMap.hasNext()); + } + } + assertFalse(oldInfoIt.hasNext()); + } + + @Test + public void testGetTermVectorsInt() { + try { + TestUtils.compareTerms(unsortedIndexReader.getTermVector(2, VAL), sortingIndexReader.getTermVector(0, VAL)); + TestUtils.compareTerms(unsortedIndexReader.getTermVector(1, VAL), sortingIndexReader.getTermVector(1, VAL)); + TestUtils.compareTerms(unsortedIndexReader.getTermVector(0, VAL), sortingIndexReader.getTermVector(2, VAL)); + + } catch (IOException e) { + fail("Exception"); + } + + } + + @Test + public void testNormValuesString() { + try { + assertNull(sortingIndexReader.normValues("")); + final DocValues sortedNormValues = sortingIndexReader.normValues(VAL); + final DocValues oldNormValues = atomicReader.normValues(VAL); + assertEquals(oldNormValues.getDirectSource().getInt(2), sortedNormValues.getDirectSource().getInt(0)); + assertEquals(oldNormValues.getDirectSource().getInt(1), sortedNormValues.getDirectSource().getInt(1)); + assertEquals(oldNormValues.getDirectSource().getInt(0), sortedNormValues.getDirectSource().getInt(2)); + } catch (IOException e) { + fail("Exception"); + } + + } + +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SortingSourceTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SortingSourceTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SortingSourceTest.java (working copy) @@ -0,0 +1,131 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FloatDocValuesField; +import org.apache.lucene.document.IntDocValuesField; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValues.Source; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public class SortingSourceTest extends LuceneTestCase{ + + static final IndexWriterConfig INDEX_WRITER_CONFIG = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random())); + + private SortingSource sortingSource; + private final int[] oldToNew = { 2, 1, 0 }; + AtomicReader atomicReader; + + private static final String I_VAL = "iVal"; + private static final String F_VAL = "fVal"; + + private Document doc(final Integer id) { + final Document doc = new Document(); + + doc.add(new IntDocValuesField(I_VAL, id)); + doc.add(new FloatDocValuesField(F_VAL, id)); + + return doc; + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + Directory dir = new RAMDirectory(); + final IndexWriter indexWriter = new IndexWriter(dir, INDEX_WRITER_CONFIG); + + indexWriter.addDocument(doc(0)); + indexWriter.addDocument(doc(1)); + indexWriter.addDocument(doc(2)); + indexWriter.close(); + + final DirectoryReader dirReader = DirectoryReader.open(dir); + atomicReader = new SlowCompositeReaderWrapper(dirReader); + + } + + @Test + public void testGetInt() { + try { + DocValues docValues = atomicReader.docValues(I_VAL); + Source unsortedSource = docValues.getSource(); + sortingSource = new SortingSource(docValues.getSource(), oldToNew); + + assertEquals(unsortedSource.getInt(2), sortingSource.getInt(0)); + assertEquals(unsortedSource.getInt(1), sortingSource.getInt(1)); + assertEquals(unsortedSource.getInt(0), sortingSource.getInt(2)); + + atomicReader.close(); + + } catch (IOException e) { + fail("exception"); + } + + } + + @Test + public void testGetFloat() { + try { + DocValues docValues = atomicReader.docValues(F_VAL); + Source unsortedSource = docValues.getSource(); + sortingSource = new SortingSource(docValues.getSource(), oldToNew); + atomicReader.close(); + + assertTrue(Double.compare(unsortedSource.getFloat(2), sortingSource.getFloat(0)) == 0); + assertTrue(Double.compare(unsortedSource.getFloat(1), sortingSource.getFloat(1)) == 0); + assertTrue(Double.compare(unsortedSource.getFloat(0), sortingSource.getFloat(2)) == 0); + + } catch (IOException e) { + fail("exception"); + } + + } + + @Test + public void testGetByteRef() { + try { + DocValues docValues = atomicReader.docValues(F_VAL); + Source unsortedSource = docValues.getSource(); + sortingSource = new SortingSource(docValues.getSource(), oldToNew); + atomicReader.close(); + + assertEquals(unsortedSource.getBytes(2, new BytesRef()), sortingSource.getBytes(0, new BytesRef())); + assertEquals(unsortedSource.getBytes(1, new BytesRef()), sortingSource.getBytes(1, new BytesRef())); + assertEquals(unsortedSource.getBytes(0, new BytesRef()), sortingSource.getBytes(2, new BytesRef())); + + } catch (IOException e) { + fail("exception"); + } + + } +} Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SortingTermEnumTest.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SortingTermEnumTest.java (revision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SortingTermEnumTest.java (working copy) @@ -0,0 +1,195 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; +/*** + * + * + *@lucene.experimental + */ +public class SortingTermEnumTest extends LuceneTestCase{ + + static final IndexWriterConfig INDEX_WRITER_CONFIG = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(new Random())); + + private TermsEnum unsortedTermsEnum; + private SortingTermEnum sortingTermEnum; + private final int[] oldToNew = { 2, 1, 0 }; + AtomicReader atomicReader; + + private static final String VAL = "Val"; + + + private Document doc(final Integer id) { + final Document doc = new Document(); + FieldType ft = new FieldType(); + ft.setIndexed(true); + ft.setStored(true); + for ( int i=0; i <= id; i++) { + doc.add(new Field(VAL,Integer.toString(i), ft)); + } + return doc; + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + Directory dir = new RAMDirectory(); + final IndexWriter indexWriter = new IndexWriter(dir, INDEX_WRITER_CONFIG); + + indexWriter.addDocument(doc(0)); + indexWriter.addDocument(doc(1)); + indexWriter.addDocument(doc(2)); + indexWriter.close(); + + final DirectoryReader dirReader = DirectoryReader.open(dir); + atomicReader = new SlowCompositeReaderWrapper(dirReader); + unsortedTermsEnum = atomicReader.terms(VAL).iterator(null); + sortingTermEnum = new SortingTermEnum(atomicReader.terms(VAL).iterator(null), oldToNew); + } + + + @Test + public void testDocFreq() throws IOException { + TermsEnum unsortedEnum = atomicReader.fields().terms(VAL).iterator(null); + for (int i = 0; i < 3 ; i++) { + sortingTermEnum.next(); + unsortedEnum.next(); + assertEquals(unsortedEnum.docFreq(), sortingTermEnum.docFreq()); + } + assertNull(sortingTermEnum.next()); + assertNull(unsortedEnum.next()); + } + + @Test + public void testTotalTermFreq() { + try { + TermsEnum unsortedTermEnum = atomicReader.terms(VAL).iterator(null); + unsortedTermEnum.next(); + sortingTermEnum.next(); + assertEquals(unsortedTermEnum.totalTermFreq(), sortingTermEnum.totalTermFreq()); + } catch (IOException e) { + fail("Exception"); + } + } + + + @Test + public void testDocs() { + try { + BytesRef term = sortingTermEnum.next(); + while (term != null) { + DocsEnum sortedDocs = sortingTermEnum.docs(null, null); + for ( int j = 0; j - - - - - Building ${ant.project.name}... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -