Index: contrib/misc/src/test/org/apache/lucene/index/TestIndexSorter.java =================================================================== --- contrib/misc/src/test/org/apache/lucene/index/TestIndexSorter.java (revision 0) +++ contrib/misc/src/test/org/apache/lucene/index/TestIndexSorter.java (revision 0) @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; + +public class TestIndexSorter extends LuceneTestCase { + + private static final int NUM_DOCS = 4; + private String[] fieldNames = new String[] { "id", "url", "site", "content", + "host", "anchor", "boost" }; + + Directory inputDir = null; + Directory outputDir = null; + + public void setUp() throws Exception { + super.setUp(); + // create test index + inputDir = new RAMDirectory(); + IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, + new WhitespaceAnalyzer(Version.LUCENE_40)); + IndexWriter writer = new IndexWriter(inputDir, cfg); + // create test documents + for (int i = 0; i < NUM_DOCS; i++) { + Document doc = new Document(); + for (int k = 0; k < fieldNames.length; k++) { + Field f; + Store s; + Index ix; + TermVector tv = TermVector.NO; + String val = null; + if (fieldNames[k].equals("id")) { + s = Store.YES; + ix = Index.NOT_ANALYZED; + val = String.valueOf(i); + } else if (fieldNames[k].equals("host")) { + s = Store.YES; + ix = Index.NOT_ANALYZED; + val = "www.example" + i + ".com"; + } else if (fieldNames[k].equals("site")) { + s = Store.NO; + ix = Index.NOT_ANALYZED; + val = "www.example" + i + ".com"; + } else if (fieldNames[k].equals("content")) { + s = Store.NO; + ix = Index.ANALYZED; + tv = TermVector.YES; + val = "This is the content of the " + i + "-th document."; + } else if (fieldNames[k].equals("boost")) { + s = Store.YES; + ix = Index.NO; + float boost = (float) i; + val = String.valueOf(boost); + } else { + s = Store.YES; + ix = Index.ANALYZED; + if (fieldNames[k].equals("anchor")) { + val = "anchors to " + i + "-th page."; + } else if (fieldNames[k].equals("url")) { + val = "http://www.example" + i + ".com/" + i + ".html"; + } + } + f = new Field(fieldNames[k], val, s, ix, tv); + doc.add(f); + } + writer.addDocument(doc); + } + writer.optimize(); + writer.close(); + outputDir = new RAMDirectory(); + } + + public void testSorting() throws Exception { + IndexSorter sorter = new IndexSorter(); + sorter.sort(inputDir, outputDir, "boost"); + + // read back documents + IndexReader reader = IndexReader.open(outputDir); + assertEquals(reader.numDocs(), NUM_DOCS); + for (int i = 0; i < reader.maxDoc(); i++) { + Document doc = reader.document(i); + Field f = doc.getField("content"); + assertNull(f); + String boost = doc.get("boost"); + int origId = NUM_DOCS - i - 1; + String cmp = String.valueOf((float) origId); + assertEquals(cmp, boost); + // check that vectors are in sync + TermFreqVector tfv = reader.getTermFreqVector(i, "content"); + assertTrue(tfv.indexOf(new BytesRef(origId + "-th")) != -1); + } + reader.close(); + } + +} Index: contrib/misc/src/java/org/apache/lucene/index/IndexSorter.java =================================================================== --- contrib/misc/src/java/org/apache/lucene/index/IndexSorter.java (revision 0) +++ contrib/misc/src/java/org/apache/lucene/index/IndexSorter.java (revision 0) @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.document.MapFieldSelector; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.Version; + +/** + * Sort an index by document importance factor. Higher scoring documents are + * assigned smaller document numbers. Document weights are obtained from a + * specified field, which has to be single-valued and stored, with string value + * that represents a float number. Stored fields in the output index remain + * consistent, i.e. both stored fields and postings are renumbered in sync. + */ +public class IndexSorter { + private static final Logger LOG = Logger.getLogger(IndexSorter.class + .getName()); + + private static class SortingReader extends FilterIndexReader { + + private int[] oldToNew; + private int[] newToOld; + + public SortingReader(IndexReader oldReader, int[] oldToNew) { + super(oldReader); + this.oldToNew = oldToNew; + + this.newToOld = new int[oldReader.maxDoc()]; + int oldDoc = 0; + while (oldDoc < oldToNew.length) { + int newDoc = oldToNew[oldDoc]; + if (newDoc != -1) { + newToOld[newDoc] = oldDoc; + } + oldDoc++; + } + } + + @Override + public Bits getDeletedDocs() { + final Bits deletedDocs = super.getDeletedDocs(); + + if (deletedDocs == null) + return null; + + return new Bits() { + @Override + public boolean get(int index) { + return deletedDocs.get(newToOld[index]); + } + + @Override + public int length() { + return deletedDocs.length(); + } + }; + } + + @Override + public TermFreqVector[] getTermFreqVectors(int docNumber) + throws IOException { + return super.getTermFreqVectors(newToOld[docNumber]); + } + + @Override + public TermFreqVector getTermFreqVector(int docNumber, String field) + throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void getTermFreqVector(int docNumber, String field, + TermVectorMapper mapper) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void getTermFreqVector(int docNumber, TermVectorMapper mapper) + throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Document document(int n) throws IOException { + return document(n, null); + } + + @Override + public Document document(int n, FieldSelector fieldSelector) + throws CorruptIndexException, IOException { + return super.document(newToOld[n], fieldSelector); + } + + @Override + protected void doUndeleteAll() throws CorruptIndexException, IOException { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] norms(String f) throws IOException { + byte[] oldNorms = super.norms(f); + byte[] newNorms = new byte[oldNorms.length]; + int newDoc; + + for (int oldDoc = 0; oldDoc < oldNorms.length; oldDoc++) { + newDoc = oldToNew[oldDoc]; + if (newDoc >= 0) { + newNorms[newDoc] = oldNorms[oldDoc]; + } + } + + return newNorms; + } + + @Override + protected void doSetNorm(int d, String f, byte b) + throws CorruptIndexException, IOException { + throw new UnsupportedOperationException(); + } + + @Override + protected void doDelete(int n) throws CorruptIndexException, IOException { + throw new UnsupportedOperationException(); + } + + @Override + protected void doCommit(Map commitUserData) + throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public IndexReader[] getSequentialSubReaders() { + return null; + } + + @Override + public ReaderContext getTopReaderContext() { + return null; + } + } + + private static class DocScore implements Comparable { + private int oldDoc; + private float score; + + public int compareTo(DocScore that) { // order by score, oldDoc + if (this.score == that.score) { + return this.oldDoc - that.oldDoc; + } else { + return this.score < that.score ? 1 : -1; + } + } + + public String toString() { + return "oldDoc=" + oldDoc + ",score=" + score; + } + } + + public IndexSorter() { + + } + + public void sort(Directory input, Directory output, String field) + throws IOException { + LOG.info("IndexSorter: starting."); + long start = System.currentTimeMillis(); + IndexReader reader = new SlowMultiReaderWrapper(IndexReader.open(input, + true)); + + SortingReader sorter = new SortingReader(reader, oldToNew(reader, field)); + IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, + new WhitespaceAnalyzer(Version.LUCENE_40)); + IndexWriter writer = new IndexWriter(output, cfg); + writer.addIndexes(new IndexReader[] { sorter }); + writer.close(); + long end = System.currentTimeMillis(); + LOG.info("IndexSorter: done, " + (end - start) + " total milliseconds"); + } + + private static int[] oldToNew(IndexReader reader, String field) + throws IOException { + int readerMax = reader.maxDoc(); + DocScore[] newToOld = new DocScore[readerMax]; + FieldSelector fSel = new MapFieldSelector(field); + Bits deletedDocs = reader.getDeletedDocs(); + + for (int oldDoc = 0; oldDoc < readerMax; oldDoc++) { + float score; + if (deletedDocs != null && deletedDocs.get(oldDoc)) { + score = 0.0f; + } else { + Document d = reader.document(oldDoc, fSel); + try { + score = Float.parseFloat(d.get(field)); + } catch (Exception e) { + LOG.warning("unable to parse '" + d.get(field) + + "' as floating point number"); + score = 0.0f; + } + } + DocScore docScore = new DocScore(); + docScore.oldDoc = oldDoc; + docScore.score = score; + newToOld[oldDoc] = docScore; + } + Arrays.sort(newToOld); + + int[] oldToNew = new int[readerMax]; + for (int newDoc = 0; newDoc < readerMax; newDoc++) { + DocScore docScore = newToOld[newDoc]; + oldToNew[docScore.oldDoc] = newDoc; + } + return oldToNew; + } + + /** */ + public static void main(String[] args) throws Exception { + Directory input, output; + String field; + + String usage = "IndexSorter "; + + if (args.length < 3) { + System.err.println("Usage: " + usage); + System.exit(-1); + } + + input = FSDirectory.open(new File(args[0])); + File out = new File(args[1]); + if (!out.exists()) + out.mkdirs(); + output = FSDirectory.open(out); + field = args[2]; + IndexSorter sorter = new IndexSorter(); + try { + sorter.sort(input, output, field); + } catch (Exception e) { + LOG.log(Level.WARNING, "IndexSorter: " + e, e); + } + } +}