Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x:r1141060 Property changes on: solr ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr:r1141060 Property changes on: solr/NOTICE.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/NOTICE.txt:r1141060 Property changes on: solr/contrib ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/contrib:r1141060 Property changes on: solr/LICENSE.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/LICENSE.txt:r1141060 Property changes on: solr/testlogging.properties ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/testlogging.properties:r1141060 Property changes on: solr/site ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/site:r1141060 Property changes on: solr/common-build.xml ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/common-build.xml:r1141060 Property changes on: solr/lib ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/lib:r1141060 Property changes on: solr/CHANGES.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/CHANGES.txt:r1141060 Property changes on: solr/src ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/src:r1141060 Property changes on: solr/example ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/example:r1141060 Property changes on: solr/README.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/README.txt:r1141060 Property changes on: solr/client ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/client:r1141060 Property changes on: solr/build.xml ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/solr/build.xml:r1141060 Property changes on: lucene ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/lucene:r1141060 Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java (working copy) @@ -4,7 +4,7 @@ import java.util.Arrays; import java.util.List; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java (working copy) @@ -5,7 +5,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java (working copy) @@ -4,14 +4,16 @@ import java.util.Iterator; import java.util.List; -import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; @@ -19,6 +21,7 @@ import org.apache.lucene.store.Directory; import org.junit.Test; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.search.MultiCollector; import org.apache.lucene.facet.FacetTestUtils; @@ -87,8 +90,8 @@ // Obtain facets results and hand-test them assertCorrectResults(facetsCollector); - TermDocs td = ir.termDocs(new Term("$facets", "$fulltree$")); - assertTrue(td.next()); + DocsEnum td = MultiFields.getTermDocsEnum(ir, MultiFields.getDeletedDocs(ir), "$facets", new BytesRef("$fulltree$")); + assertTrue(td.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); tr.close(); ir.close(); @@ -183,10 +186,8 @@ } private void assertPostingListExists(String field, String text, IndexReader ir) throws IOException { - TermDocs td; - Term term = new Term(field, text); - td = ir.termDocs(term); - assertTrue(td.next()); + DocsEnum de = MultiFields.getTermDocsEnum(ir, null, field, new BytesRef(text)); + assertTrue(de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); } @Test Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java (working copy) @@ -4,7 +4,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java (working copy) @@ -15,7 +15,7 @@ import org.junit.Test; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.facet.enhancements.EnhancementsDocumentBuilder; import org.apache.lucene.facet.enhancements.association.AssociationEnhancement; Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java (working copy) @@ -3,7 +3,7 @@ import java.io.IOException; import java.util.ArrayList; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/index/FacetsPayloadProcessorProviderTest.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/index/FacetsPayloadProcessorProviderTest.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/index/FacetsPayloadProcessorProviderTest.java (working copy) @@ -4,7 +4,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java (working copy) @@ -4,7 +4,7 @@ import java.util.Arrays; import java.util.List; -import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java (working copy) @@ -1,6 +1,6 @@ package org.apache.lucene.facet.enhancements.association; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/FacetTestBase.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (working copy) @@ -11,25 +11,28 @@ import org.apache.lucene.DocumentBuilder.DocumentBuilderException; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field.TermVector; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.facet.index.CategoryDocumentBuilder; import org.apache.lucene.facet.index.params.CategoryListParams; @@ -247,24 +250,24 @@ Map res = new HashMap(); HashSet handledTerms = new HashSet(); for (CategoryListParams clp : iParams.getAllCategoryListParams()) { - Term baseTerm = clp.getTerm().createTerm(""); + Term baseTerm = new Term(clp.getTerm().field()); if (!handledTerms.add(baseTerm)) { continue; // already handled this term (for another list) } - TermEnum te = indexReader.terms(baseTerm); - while (te.next()) { - Term t = te.term(); - if (!t.field().equals(baseTerm.field())) { - break; // hit a different field - } - TermDocs tp = indexReader.termDocs(t); + Terms terms = MultiFields.getTerms(indexReader, baseTerm.field()); + if (terms == null) { + continue; + } + Bits deletedDocs = MultiFields.getDeletedDocs(indexReader); + TermsEnum te = terms.iterator(); + DocsEnum de = null; + while (te.next() != null) { + de = te.docs(deletedDocs, de); int cnt = 0; - while (tp.next()) { - if (!indexReader.isDeleted(tp.doc())) { // ignore deleted docs - cnt++; - } + while (de.nextDoc() != DocsEnum.NO_MORE_DOCS) { + cnt++; } - res.put(new CategoryPath(t.text().split(delim)), cnt); + res.put(new CategoryPath(te.term().utf8ToString().split(delim)), cnt); } } return res; Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java (working copy) @@ -2,7 +2,7 @@ import java.io.IOException; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; @@ -10,6 +10,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; @@ -18,6 +19,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.util.OpenBitSetDISI; import org.junit.Test; @@ -154,11 +156,12 @@ // now make sure the documents in the complement set are not deleted // and not in the original result set ScoredDocIDsIterator compIterator = complementSet.iterator(); + Bits deleted = MultiFields.getDeletedDocs(reader); while (compIterator.next()) { int docNum = compIterator.getDocID(); assertFalse( "Complement-Set must not contain deleted documents (doc="+docNum+")", - reader.isDeleted(docNum)); + deleted != null && deleted.get(docNum)); assertFalse( "Complement-Set must not contain deleted documents (doc="+docNum+")", docFactory.markedDeleted(docNum)); Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/lucene/TestIndexClose.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/lucene/TestIndexClose.java (revision 1141060) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/lucene/TestIndexClose.java (working copy) @@ -4,7 +4,7 @@ import java.util.HashSet; import java.util.Set; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FilterIndexReader; import org.apache.lucene.index.IndexReader; Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java (working copy) @@ -2,9 +2,13 @@ import java.io.IOException; +import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermPositions; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -35,7 +39,7 @@ protected byte[] buffer; protected int payloadLength; - TermPositions tp; + DocsAndPositionsEnum tp; private boolean hasMore; @@ -47,7 +51,9 @@ public PayloadIterator(IndexReader indexReader, Term term, byte[] buffer) throws IOException { this.buffer = buffer; - this.tp = indexReader.termPositions(term); + // nocommit + Bits deletedDocs = MultiFields.getDeletedDocs(indexReader); + this.tp = MultiFields.getTermPositionsEnum(indexReader, deletedDocs, term.field(), term.bytes()); } /** @@ -56,7 +62,7 @@ * (no setdoc() will never return true). */ public boolean init() throws IOException { - hasMore = tp.next(); + hasMore = tp != null && tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS; return hasMore; } @@ -72,20 +78,20 @@ return false; } - if (tp.doc() > docId) { + if (tp.docID() > docId) { return false; } // making sure we have the requested document - if (tp.doc() < docId) { + if (tp.docID() < docId) { // Skipping to requested document - if (!tp.skipTo(docId)) { + if (tp.advance(docId) == DocIdSetIterator.NO_MORE_DOCS) { this.hasMore = false; return false; } // If document not found (skipped to much) - if (tp.doc() != docId) { + if (tp.docID() != docId) { return false; } } @@ -93,18 +99,25 @@ // Prepare for payload extraction tp.nextPosition(); - this.payloadLength = tp.getPayloadLength(); - if (this.payloadLength == 0) { + // nocommit -- API change vs 3.x? + if (!tp.hasPayload()) { return false; } + BytesRef br = tp.getPayload(); + + if (br == null || br.length == 0) { + return false; + } + + this.payloadLength = br.length; + if (this.payloadLength > this.buffer.length) { // Growing if necessary. this.buffer = new byte[this.payloadLength * 2 + 1]; } // Loading the payload - tp.getPayload(this.buffer, 0); - + System.arraycopy(br.bytes, br.offset, this.buffer, 0, payloadLength); return true; } Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java (working copy) @@ -4,6 +4,7 @@ import java.util.List; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.search.Collector; import org.apache.lucene.search.Scorer; @@ -124,8 +125,8 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { - scoreDocIdCollector.setNextReader(reader, docBase); + public void setNextReader(AtomicReaderContext context) throws IOException { + scoreDocIdCollector.setNextReader(context); } @Override Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java (working copy) @@ -3,6 +3,7 @@ import java.io.IOException; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.search.Collector; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; @@ -216,8 +217,8 @@ } @Override - public void setNextReader(IndexReader reader, int base) throws IOException { - this.docBase = base; + public void setNextReader(AtomicReaderContext context) throws IOException { + this.docBase = context.docBase; } } Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java (working copy) @@ -525,8 +525,8 @@ private ACComparator merger; public AggregatedCategoryHeap(int size, ACComparator merger) { + super(size); this.merger = merger; - initialize(size); } @Override @@ -539,8 +539,8 @@ private static class ResultNodeHeap extends PriorityQueue { private ACComparator merger; public ResultNodeHeap(int size, ACComparator merger) { + super(size); this.merger = merger; - initialize(size); } @Override Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java (working copy) @@ -3,8 +3,10 @@ import java.io.IOException; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.util.Bits; import org.apache.lucene.facet.search.DrillDown; import org.apache.lucene.facet.search.ScoredDocIDs; @@ -101,7 +103,9 @@ CategoryPath catPath = fresNode.getLabel(taxonomyReader); // force labeling Term drillDownTerm = DrillDown.term(searchParams, catPath); - int updatedCount = countIntersection(indexReader.termDocs(drillDownTerm), + // nocommit + Bits deletedDocs = MultiFields.getDeletedDocs(indexReader); + int updatedCount = countIntersection(MultiFields.getTermDocsEnum(indexReader, deletedDocs, drillDownTerm.field(), drillDownTerm.bytes()), docIds.iterator()); fresNode.setValue(updatedCount); @@ -112,38 +116,39 @@ * documents in a certain category) and a DocIdSetIterator (list of documents * matching a query). */ - private static int countIntersection(TermDocs p1, ScoredDocIDsIterator p2) + private static int countIntersection(DocsEnum p1, ScoredDocIDsIterator p2) throws IOException { // The documentation of of both TermDocs and DocIdSetIterator claim // that we must do next() before doc(). So we do, and if one of the // lists is empty, obviously return 0; - if (!p1.next()) { + // nocommit + if (p1 == null || p1.nextDoc() == DocsEnum.NO_MORE_DOCS) { return 0; } if (!p2.next()) { return 0; } - int d1 = p1.doc(); + int d1 = p1.docID(); int d2 = p2.getDocID(); int count = 0; for (;;) { if (d1 == d2) { ++count; - if (!p1.next()) { + if (p1.nextDoc() == DocsEnum.NO_MORE_DOCS) { break; // end of list 1, nothing more in intersection } - d1 = p1.doc(); + d1 = p1.docID(); if (!advance(p2, d1)) { break; // end of list 2, nothing more in intersection } d2 = p2.getDocID(); } else if (d1 < d2) { - if (!p1.skipTo(d2)) { + if (p1.advance(d2) == DocsEnum.NO_MORE_DOCS) { break; // end of list 1, nothing more in intersection } - d1 = p1.doc(); + d1 = p1.docID(); } else /* d1>d2 */ { if (!advance(p2, d1)) { break; // end of list 2, nothing more in intersection Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/DrillDown.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/DrillDown.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/DrillDown.java (working copy) @@ -49,7 +49,7 @@ CategoryListParams clp = iParams.getCategoryListParams(path); char[] buffer = new char[path.charsNeededForFullPath()]; iParams.drillDownTermText(path, buffer); - return clp.getTerm().createTerm(String.valueOf(buffer)); + return new Term(clp.getTerm().field(), String.valueOf(buffer)); } /** Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java (working copy) @@ -141,7 +141,7 @@ public CategoryListIterator createCategoryListIterator(IndexReader reader, int partition) throws IOException { String categoryListTermStr = PartitionsUtils.partitionName(this, partition); - Term payloadTerm = term.createTerm(categoryListTermStr); + Term payloadTerm = new Term(term.field(), categoryListTermStr); return new PayloadIntDecodingIterator(reader, payloadTerm, createEncoder().createMatchingDecoder()); } Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadProcessorProvider.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadProcessorProvider.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadProcessorProvider.java (working copy) @@ -14,6 +14,7 @@ import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.taxonomy.lucene.LuceneTaxonomyWriter.OrdinalMap; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.encoding.IntDecoder; import org.apache.lucene.util.encoding.IntEncoder; @@ -135,8 +136,9 @@ } @Override - public PayloadProcessor getProcessor(Term term) throws IOException { - CategoryListParams params = termMap.get(term); + public PayloadProcessor getProcessor(String field, BytesRef bytes) throws IOException { + // nocommit, don't create terms + CategoryListParams params = termMap.get(new Term(field, bytes)); if (params == null) { return null; } @@ -162,15 +164,10 @@ decoder = encoder.createMatchingDecoder(); this.ordinalMap = ordinalMap; } - - @Override - public int payloadLength() throws IOException { - return os.size(); - } @Override - public byte[] processPayload(byte[] payload, int start, int length) throws IOException { - InputStream is = new ByteArrayInputStream(payload, start, length); + public void processPayload(BytesRef payload) throws IOException { + InputStream is = new ByteArrayInputStream(payload.bytes, payload.offset, payload.length); decoder.reInit(is); os.reset(); encoder.reInit(os); @@ -180,7 +177,11 @@ encoder.encode(newOrdinal); } encoder.close(); - return os.toByteArray(); + // nocommit + byte out[] = os.toByteArray(); + payload.bytes = out; + payload.offset = 0; + payload.length = out.length; } } Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/util/RandomSample.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/util/RandomSample.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/util/RandomSample.java (working copy) @@ -6,7 +6,7 @@ import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; @@ -346,8 +346,7 @@ * @param size The number of elements to retain. */ public IntPriorityQueue(int size) { - super(); - this.initialize(size); + super(size); } /** Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java (working copy) @@ -4,8 +4,10 @@ import java.util.Arrays; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.util.OpenBitSetDISI; @@ -77,11 +79,13 @@ if (!reader.hasDeletions()) { return; // return immediately } + + Bits bits = MultiFields.getDeletedDocs(reader); DocIdSetIterator it = set.iterator(); int doc = DocIdSetIterator.NO_MORE_DOCS; while ((doc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (reader.isDeleted(doc)) { + if (bits.get(doc)) { set.fastClear(doc); } } @@ -335,6 +339,7 @@ @Override public DocIdSetIterator iterator() throws IOException { return new DocIdSetIterator() { + final Bits deletedDocs = MultiFields.getDeletedDocs(reader); private int next = -1; @Override @@ -354,7 +359,7 @@ public int nextDoc() throws IOException { do { ++next; - } while (next < maxDoc && reader.isDeleted(next)); + } while (next < maxDoc && deletedDocs != null && deletedDocs.get(next)); return next < maxDoc ? next : NO_MORE_DOCS; } Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/util/ResultSortUtils.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/util/ResultSortUtils.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/util/ResultSortUtils.java (working copy) @@ -68,7 +68,7 @@ private static class MinValueHeap extends PriorityQueue implements Heap { public MinValueHeap(int size) { - initialize(size); + super(size); } @Override @@ -88,7 +88,7 @@ private static class MaxValueHeap extends PriorityQueue implements Heap { public MaxValueHeap(int size) { - initialize(size); + super(size); } @Override @@ -108,7 +108,7 @@ private static class MinOrdinalHeap extends PriorityQueue implements Heap { public MinOrdinalHeap(int size) { - initialize(size); + super(size); } @Override @@ -121,7 +121,7 @@ private static class MaxOrdinalHeap extends PriorityQueue implements Heap { public MaxOrdinalHeap(int size) { - initialize(size); + super(size); } @Override Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/lucene/LuceneTaxonomyWriter.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/lucene/LuceneTaxonomyWriter.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/lucene/LuceneTaxonomyWriter.java (working copy) @@ -11,7 +11,7 @@ import java.io.IOException; import java.util.Map; -import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -20,18 +20,23 @@ import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.LogByteSizeMergePolicy; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.TermsEnum.SeekStatus; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.SimpleFSLockFactory; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.apache.lucene.facet.taxonomy.CategoryPath; @@ -356,9 +361,11 @@ reader = openReader(); } - TermDocs docs = reader.termDocs(new Term(Consts.FULL, categoryPath - .toString(delimiter))); - if (!docs.next()) { + // nocommit + Bits deletedDocs = MultiFields.getDeletedDocs(reader); + DocsEnum docs = MultiFields.getTermDocsEnum(reader, deletedDocs, Consts.FULL, + new BytesRef(categoryPath.toString(delimiter))); + if (docs == null || docs.nextDoc() == DocsEnum.NO_MORE_DOCS) { return -1; // category does not exist in taxonomy } // Note: we do NOT add to the cache the fact that the category @@ -366,8 +373,8 @@ // method is just before we actually add this category. If // in the future this usage changes, we should consider caching // the fact that the category is not in the taxonomy. - addToCache(categoryPath, docs.doc()); - return docs.doc(); + addToCache(categoryPath, docs.docID()); + return docs.docID(); } /** @@ -391,13 +398,14 @@ if (reader == null) { reader = openReader(); } - TermDocs docs = reader.termDocs(new Term(Consts.FULL, categoryPath - .toString(delimiter, prefixLen))); - if (!docs.next()) { + Bits deletedDocs = MultiFields.getDeletedDocs(reader); + DocsEnum docs = MultiFields.getTermDocsEnum(reader, deletedDocs, Consts.FULL, + new BytesRef(categoryPath.toString(delimiter, prefixLen))); + if (docs == null || docs.nextDoc() == DocsEnum.NO_MORE_DOCS) { return -1; // category does not exist in taxonomy } - addToCache(categoryPath, prefixLen, docs.doc()); - return docs.doc(); + addToCache(categoryPath, prefixLen, docs.docID()); + return docs.docID(); } // TODO (Facet): addCategory() is synchronized. This means that if indexing is @@ -667,30 +675,30 @@ } CategoryPath cp = new CategoryPath(); - TermDocs td = reader.termDocs(); - Term fullPathTerm = new Term(Consts.FULL); - String field = fullPathTerm.field(); // needed so we can later use != - TermEnum terms = reader.terms(fullPathTerm); + Terms terms = MultiFields.getTerms(reader, Consts.FULL); // The check is done here to avoid checking it on every iteration of the // below loop. A null term wlil be returned if there are no terms in the // lexicon, or after the Consts.FULL term. However while the loop is // executed we're safe, because we only iterate as long as there are next() // terms. - if (terms.term() != null) { - do { - Term t = terms.term(); - if (t.field() != field) break; + if (terms != null) { + TermsEnum termsEnum = terms.iterator(); + Bits deletedDocs = MultiFields.getDeletedDocs(reader); + DocsEnum docsEnum = null; + while (termsEnum.next() != null) { + BytesRef t = termsEnum.term(); // Since we guarantee uniqueness of categories, each term has exactly // one document. Also, since we do not allow removing categories (and // hence documents), there are no deletions in the index. Therefore, it // is sufficient to call next(), and then doc(), exactly once with no // 'validation' checks. - td.seek(t); - td.next(); + docsEnum = termsEnum.docs(deletedDocs, docsEnum); + docsEnum.nextDoc(); cp.clear(); - cp.add(t.text(), delimiter); - cache.put(cp, td.doc()); - } while (terms.next()); + // nocommit + cp.add(t.utf8ToString(), delimiter); + cache.put(cp, docsEnum.docID()); + } } cacheIsComplete = true; @@ -745,13 +753,20 @@ // to open a reader, and when not, we'll be opening a new reader instead // of using the existing "reader" object: IndexReader mainreader = openReader(); - TermEnum mainte = mainreader.terms(new Term(Consts.FULL)); + // nocommit + Terms terms = MultiFields.getTerms(mainreader, Consts.FULL); + assert terms != null; // nocommit + TermsEnum mainte = terms.iterator(); + DocsEnum mainde = null; IndexReader[] otherreaders = new IndexReader[taxonomies.length]; - TermEnum[] othertes = new TermEnum[taxonomies.length]; + TermsEnum[] othertes = new TermsEnum[taxonomies.length]; + DocsEnum[] otherdocsEnum = new DocsEnum[taxonomies.length]; // just for reuse for (int i=0; i= i (this is an // invariant kept throughout this loop) - if (positions.doc()==i) { + if (positions.docID()==i) { if (positions.freq() == 0) { // shouldn't happen throw new CorruptIndexException( "Missing parent data for category "+i); @@ -120,7 +124,7 @@ // increment we added originally, so we get here the right numbers: prefetchParentOrdinal[i] = positions.nextPosition(); - if (!positions.next()) { + if (positions.nextDoc() == DocsAndPositionsEnum.NO_MORE_DOCS) { if ( i+1 < num ) { throw new CorruptIndexException( "Missing parent data for category "+(i+1)); @@ -128,12 +132,9 @@ break; } } else { // this shouldn't happen - throw new CorruptIndexException( - "Missing parent data for category "+i); - } + throw new CorruptIndexException( + "Missing parent data for category "+i); } - } finally { - positions.close(); // to be on the safe side. } } Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/lucene/LuceneTaxonomyReader.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/lucene/LuceneTaxonomyReader.java (revision 1141060) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/lucene/LuceneTaxonomyReader.java (working copy) @@ -12,13 +12,16 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.collections.LRUHashMap; /** @@ -210,9 +213,12 @@ int ret = TaxonomyReader.INVALID_ORDINAL; try { indexReaderLock.readLock().lock(); - TermDocs docs = indexReader.termDocs(new Term(Consts.FULL, path)); - if (docs.next()) { - ret = docs.doc(); + // nocommit + Bits deletedDocs = MultiFields.getDeletedDocs(indexReader); + // nocommit + DocsEnum docs = MultiFields.getTermDocsEnum(indexReader, deletedDocs, Consts.FULL, new BytesRef(path)); + if (docs != null && docs.nextDoc() != DocsEnum.NO_MORE_DOCS) { + ret = docs.docID(); } } finally { indexReaderLock.readLock().unlock(); Index: lucene/contrib/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleUtils.java =================================================================== --- lucene/contrib/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleUtils.java (revision 1141060) +++ lucene/contrib/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleUtils.java (working copy) @@ -4,7 +4,7 @@ import java.util.List; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.facet.example.ExampleUtils; import org.apache.lucene.facet.taxonomy.CategoryPath; Index: lucene/contrib/facet/build.xml =================================================================== --- lucene/contrib/facet/build.xml (revision 1141060) +++ lucene/contrib/facet/build.xml (working copy) @@ -25,16 +25,27 @@ + + + + + + + + + + @@ -46,8 +57,14 @@ destfile="${build.dir}/${final.name}-examples.jar" title="Lucene Search Engine: ${ant.project.name}-examples" /> + + + + + + - + Index: lucene/contrib/CHANGES.txt =================================================================== --- lucene/contrib/CHANGES.txt (revision 1141095) +++ lucene/contrib/CHANGES.txt (working copy) @@ -65,7 +65,13 @@ * LUCENE-3234: provide a limit on phrase analysis in FastVectorHighlighter for highlighting speed up. Use FastVectorHighlighter.setPhraseLimit() to set limit (e.g. 5000). (Mike Sokolov via Koji Sekiguchi) - + + * LUCENE-3079: a new facet module which provides faceted indexing & search + capabilities. It allows managing a taxonomy of categories, and index them + with documents. It also provides search API for aggregating (e.g. count) + the weights of the categories that are relevant to the search results. + (Shai Erera) + * LUCENE-3171: Added BlockJoinQuery and BlockJoinCollector, under the new contrib/join module, to enable searches that require joining between parent and child documents. Joined (children + parent) Property changes on: lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/TestNumericRangeFilterBuilder.java ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/TestNumericRangeFilterBuilder.java:r1141060 Property changes on: lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java:r1141060 Property changes on: dev-tools/idea/lucene/contrib ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/branch_3x/dev-tools/idea/lucene/contrib:r1141060 Index: dev-tools/eclipse/dot.classpath =================================================================== --- dev-tools/eclipse/dot.classpath (revision 1141095) +++ dev-tools/eclipse/dot.classpath (working copy) @@ -5,6 +5,9 @@ + + +