Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java (working copy) @@ -35,10 +35,14 @@ import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LogByteSizeMergePolicy; +import org.apache.lucene.index.LogMergePolicy; import org.apache.lucene.index.Payload; import org.apache.lucene.index.Term; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Version; import org.apache.tika.io.IOExceptionWithCause; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,9 +90,6 @@ /** Compound file flag */ private boolean useCompoundFile = true; - /** maxFieldLength config parameter */ - private int maxFieldLength = SearchIndex.DEFAULT_MAX_FIELD_LENGTH; - /** termInfosIndexDivisor config parameter */ private int termInfosIndexDivisor = SearchIndex.DEFAULT_TERM_INFOS_INDEX_DIVISOR; @@ -142,8 +143,7 @@ this.isExisting = IndexReader.indexExists(directory); if (!isExisting) { - indexWriter = new IndexWriter(directory, analyzer, - IndexWriter.MaxFieldLength.LIMITED); + indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_36, analyzer)); // immediately close, now that index has been created indexWriter.close(); indexWriter = null; @@ -291,7 +291,7 @@ return readOnlyReader; } else { // reader outdated - if (readOnlyReader.getRefCount() == 1) { + if (readOnlyReader.getRefCountJr() == 1) { // not in use, except by this index // update the reader readOnlyReader.updateDeletedDocs(modifiableReader); @@ -308,7 +308,7 @@ // if we get here there is no up-to-date read-only reader if (sharedReader == null) { // create new shared reader - IndexReader reader = IndexReader.open(getDirectory(), null, true, termInfosIndexDivisor); + IndexReader reader = IndexReader.open(getDirectory(), termInfosIndexDivisor); CachingIndexReader cr = new CachingIndexReader( reader, cache, initCache); sharedReader = new SharedIndexReader(cr); @@ -345,10 +345,14 @@ indexReader = null; } if (indexWriter == null) { - indexWriter = new IndexWriter(getDirectory(), analyzer, - new IndexWriter.MaxFieldLength(maxFieldLength)); - indexWriter.setSimilarity(similarity); - indexWriter.setUseCompoundFile(useCompoundFile); + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_36, analyzer); + config.setSimilarity(similarity); + LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); + mergePolicy.setUseCompoundFile(useCompoundFile); + mergePolicy.setNoCFSRatio(1.0); + config.setMergePolicy(mergePolicy); + + indexWriter = new IndexWriter(getDirectory(), config); indexWriter.setInfoStream(STREAM_LOGGER); } return indexWriter; @@ -381,7 +385,7 @@ // optimize if requested if (optimize) { IndexWriter writer = getIndexWriter(); - writer.optimize(); + writer.forceMerge(1, true); writer.close(); indexWriter = null; } @@ -528,26 +532,13 @@ //-------------------------< properties >----------------------------------- /** - * The lucene index writer property: useCompountFile + * Whether the index writer should use the compound file format */ void setUseCompoundFile(boolean b) { useCompoundFile = b; - if (indexWriter != null) { - indexWriter.setUseCompoundFile(b); - } } /** - * The lucene index writer property: maxFieldLength - */ - void setMaxFieldLength(int maxFieldLength) { - this.maxFieldLength = maxFieldLength; - if (indexWriter != null) { - indexWriter.setMaxFieldLength(maxFieldLength); - } - } - - /** * @return the current value for termInfosIndexDivisor. */ public int getTermInfosIndexDivisor() { @@ -571,7 +562,7 @@ * @param f a lucene field. * @return the index parameter on f. */ - private Field.Index getIndexParameter(Fieldable f) { + private static Field.Index getIndexParameter(Fieldable f) { if (!f.isIndexed()) { return Field.Index.NO; } else if (f.isTokenized()) { @@ -587,7 +578,7 @@ * @param f a lucene field. * @return the term vector parameter on f. */ - private Field.TermVector getTermVectorParameter(Fieldable f) { + private static Field.TermVector getTermVectorParameter(Fieldable f) { if (f.isStorePositionWithTermVector() && f.isStoreOffsetWithTermVector()) { return Field.TermVector.WITH_POSITIONS_OFFSETS; } else if (f.isStorePositionWithTermVector()) { Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractWeight.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractWeight.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractWeight.java (working copy) @@ -77,7 +77,7 @@ starts[readers.length] = maxDoc; Scorer[] scorers = new Scorer[readers.length]; for (int i = 0; i < readers.length; i++) { - scorers[i] = scorer(readers[i], scoreDocsInOrder, topScorer); + scorers[i] = scorer(readers[i], scoreDocsInOrder, false); } return new MultiScorer(searcher.getSimilarity(), scorers, starts); Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FilterIndexReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; @@ -38,6 +39,7 @@ import org.apache.lucene.index.TermEnum; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.ReaderUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -251,6 +253,16 @@ //--------------------< FilterIndexReader overwrites >---------------------- + @Override + public IndexReader[] getSequentialSubReaders() { + return null; + } + + @Override + public FieldInfos getFieldInfos() { + return ReaderUtil.getMergedFieldInfos(in); + } + /** * Uses the {@link #docNumber2id} cache for document lookups that are only * interested in the {@link FieldSelectors#UUID}. Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingMultiIndexReader.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingMultiIndexReader.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingMultiIndexReader.java (working copy) @@ -51,11 +51,6 @@ private final DocNumberCache cache; /** - * Doc number starts for each sub reader - */ - private int[] starts; - - /** * Reference count. Every time close is called refCount is decremented. If * refCount drops to zero the underlying readers are closed as well. */ @@ -72,15 +67,10 @@ super(subReaders); this.cache = cache; this.subReaders = subReaders; - starts = new int[subReaders.length + 1]; - int maxDoc = 0; for (int i = 0; i < subReaders.length; i++) { - starts[i] = maxDoc; - maxDoc += subReaders[i].maxDoc(); OffsetReader offsetReader = new OffsetReader(subReaders[i], starts[i]); readersByCreationTick.put(subReaders[i].getCreationTick(), offsetReader); } - starts[subReaders.length] = maxDoc; } /** @@ -213,34 +203,6 @@ return -1; } - /** - * Returns the reader index for document n. - * Implementation copied from lucene MultiReader class. - * - * @param n document number. - * @return the reader index. - */ - private int readerIndex(int n) { - int lo = 0; // search starts array - int hi = subReaders.length - 1; // for first element less - - while (hi >= lo) { - int mid = (lo + hi) >> 1; - int midValue = starts[mid]; - if (n < midValue) { - hi = mid - 1; - } else if (n > midValue) { - lo = mid + 1; - } else { // found a match - while (mid + 1 < subReaders.length && starts[mid + 1] == midValue) { - mid++; // scan to last match - } - return mid; - } - } - return hi; - } - //-----------------------< OffsetTermDocs >--------------------------------- /** Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ChildAxisQuery.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ChildAxisQuery.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ChildAxisQuery.java (working copy) @@ -329,9 +329,9 @@ */ @Override public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - contextScorer = contextQuery.weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + contextScorer = contextQuery.weight(searcher).scorer(reader, scoreDocsInOrder, false); if (nameTest != null) { - nameTestScorer = new NameQuery(nameTest, version, nsMappings).weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + nameTestScorer = new NameQuery(nameTest, version, nsMappings).weight(searcher).scorer(reader, scoreDocsInOrder, false); } return new ChildAxisScorer(searcher.getSimilarity(), reader, (HierarchyResolver) reader); Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CommittableIndexReader.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CommittableIndexReader.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CommittableIndexReader.java (working copy) @@ -137,4 +137,14 @@ BitSet getDeletedDocs() { return (BitSet) deletedDocs.clone(); } + + @Override + public String toString() { + final StringBuilder buffer = new StringBuilder("CommittableIndexReader("); + buffer.append(in); + buffer.append(','); + buffer.append(modCount); + buffer.append(')'); + return buffer.toString(); + } } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DerefQuery.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DerefQuery.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DerefQuery.java (working copy) @@ -205,9 +205,9 @@ @Override public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - contextScorer = contextQuery.weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + contextScorer = contextQuery.weight(searcher).scorer(reader, scoreDocsInOrder, false); if (nameTest != null) { - nameTestScorer = new NameQuery(nameTest, version, nsMappings).weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + nameTestScorer = new NameQuery(nameTest, version, nsMappings).weight(searcher).scorer(reader, scoreDocsInOrder, false); } return new DerefScorer(searcher.getSimilarity(), reader); } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DescendantSelfAxisQuery.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DescendantSelfAxisQuery.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DescendantSelfAxisQuery.java (working copy) @@ -392,8 +392,8 @@ */ public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - contextScorer = contextQuery.weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); - subScorer = subQuery.weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + contextScorer = searcher.createNormalizedWeight(contextQuery).scorer(reader, scoreDocsInOrder, false); + subScorer = searcher.createNormalizedWeight(subQuery).scorer(reader, scoreDocsInOrder, false); HierarchyResolver resolver = (HierarchyResolver) reader; return new DescendantSelfAxisScorer(searcher.getSimilarity(), reader, resolver); } @@ -471,9 +471,14 @@ } collectContextHits(); - currentDoc = subScorer.nextDoc(); if (contextHits.isEmpty()) { currentDoc = NO_MORE_DOCS; + } else { + if (subScorer != null) { + currentDoc = subScorer.nextDoc(); + } else { + currentDoc = NO_MORE_DOCS; + } } while (currentDoc != NO_MORE_DOCS) { if (isValid(currentDoc)) { @@ -505,7 +510,9 @@ // optimize in the case of an advance to finish. // see https://issues.apache.org/jira/browse/JCR-3082 if (target == NO_MORE_DOCS) { - subScorer.advance(target); + if (subScorer != null) { + subScorer.advance(target); + } currentDoc = NO_MORE_DOCS; return currentDoc; } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/directory/FSDirectoryManager.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/directory/FSDirectoryManager.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/directory/FSDirectoryManager.java (working copy) @@ -152,7 +152,7 @@ @Override public String[] listAll() throws IOException { - File[] files = directory.getFile().listFiles(FILTER); + File[] files = directory.getDirectory().listFiles(FILTER); if (files == null) { return null; } @@ -196,7 +196,7 @@ @Override public IndexInput openInput(String name) throws IOException { IndexInput in = directory.openInput(name); - return new IndexInputLogWrapper(in); + return new IndexInputLogWrapper(name, in); } @Override @@ -208,7 +208,7 @@ public IndexInput openInput(String name, int bufferSize) throws IOException { IndexInput in = directory.openInput(name, bufferSize); - return new IndexInputLogWrapper(in); + return new IndexInputLogWrapper(name, in); } @Override @@ -222,7 +222,7 @@ } @Override - public void setLockFactory(LockFactory lockFactory) { + public void setLockFactory(LockFactory lockFactory) throws IOException { directory.setLockFactory(lockFactory); } @@ -249,7 +249,8 @@ private IndexInput in; - IndexInputLogWrapper(IndexInput in) { + IndexInputLogWrapper(String name, IndexInput in) { + super(name); this.in = in; } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/FilterSearcher.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/FilterSearcher.java (revision 0) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/FilterSearcher.java (revision 0) @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.core.query.lucene; + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.search.Weight; + +/** + * FilterSearcher wraps another Searcher and forwards all + * calls to the wrapped Searcher. + */ +class FilterSearcher extends Searcher { + + private Searcher s; + + FilterSearcher(Searcher searcher) { + this.s = searcher; + } + + @Override + public void search(Weight weight, Filter filter, Collector results) + throws IOException { + s.search(weight, filter, results); + } + + @Override + public void close() throws IOException { + s.close(); + } + + @Override + public int docFreq(Term term) throws IOException { + return s.docFreq(term); + } + + @Override + public int maxDoc() throws IOException { + return s.maxDoc(); + } + + @Override + public TopDocs search(Weight weight, Filter filter, int n) + throws IOException { + return s.search(weight, filter, n); + } + + @Override + public Document doc(int i) throws CorruptIndexException, IOException { + return s.doc(i); + } + + @Override + public Document doc(int docid, FieldSelector fieldSelector) + throws CorruptIndexException, IOException { + return s.doc(docid, fieldSelector); + } + + @Override + public Query rewrite(Query query) throws IOException { + return s.rewrite(query); + } + + @Override + public Explanation explain(Weight weight, int doc) throws IOException { + return s.explain(weight, doc); + } + + @Override + public TopFieldDocs search(Weight weight, Filter filter, int n, Sort sort) + throws IOException { + return null; + } +} Property changes on: jackrabbit-core\src\main\java\org\apache\jackrabbit\core\query\lucene\FilterSearcher.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision Rev URL Added: svn:eol-style + native Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IDField.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IDField.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IDField.java (working copy) @@ -21,6 +21,8 @@ import org.apache.jackrabbit.core.id.NodeId; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.AbstractField; +import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.FieldInfo.IndexOptions; /** * IDField implements a lucene field for the id of a node. @@ -37,7 +39,8 @@ this.isStored = true; this.isTokenized = false; this.omitNorms = true; - this.omitTermFreqAndPositions = true; + setIndexOptions(IndexOptions.DOCS_ONLY); + setStoreTermVector(TermVector.NO); } public String stringValue() { @@ -48,10 +51,6 @@ return null; } - public byte[] binaryValue() { - return null; - } - public TokenStream tokenStreamValue() { return null; } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexFormatVersion.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexFormatVersion.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexFormatVersion.java (working copy) @@ -19,6 +19,7 @@ import java.util.Collection; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.ReaderUtil; /** * This class indicates the lucene index format that is used. @@ -102,8 +103,7 @@ * index reader. */ public static IndexFormatVersion getVersion(IndexReader indexReader) { - Collection fields = indexReader.getFieldNames( - IndexReader.FieldOption.ALL); + Collection fields = ReaderUtil.getIndexedFields(indexReader); if (fields.contains(FieldNames.LOCAL_NAME) || indexReader.numDocs() == 0) { return IndexFormatVersion.V3; } else if (fields.contains(FieldNames.PROPERTIES_SET)) { Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexMigration.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexMigration.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/IndexMigration.java (working copy) @@ -33,13 +33,20 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FilterIndexReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; +import org.apache.lucene.index.LogByteSizeMergePolicy; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermEnum; import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.UpgradeIndexMergePolicy; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.ReaderUtil; +import org.apache.lucene.util.Version; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,21 +109,23 @@ // if we get here then the index must be migrated log.debug("Index requires migration {}", indexDir); - String migrationName = index.getName() + "_v2.3"; + String migrationName = index.getName() + "_v36"; if (directoryManager.hasDirectory(migrationName)) { directoryManager.delete(migrationName); } Directory migrationDir = directoryManager.getDirectory(migrationName); + final IndexWriterConfig c = new IndexWriterConfig(Version.LUCENE_36, new JackrabbitAnalyzer()); + c.setMergePolicy(new UpgradeIndexMergePolicy(new LogByteSizeMergePolicy())); + c.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); try { - IndexWriter writer = new IndexWriter(migrationDir, new JackrabbitAnalyzer(), - IndexWriter.MaxFieldLength.UNLIMITED); + IndexWriter writer = new IndexWriter(migrationDir, c); try { - IndexReader r = new MigrationIndexReader( - IndexReader.open(index.getDirectory(), true), + IndexReader r = new MigrationIndexReader(IndexReader.open(index.getDirectory()), oldSeparatorChar); try { - writer.addIndexes(new IndexReader[]{r}); + writer.addIndexes(r); + writer.forceMerge(1); writer.close(); } finally { r.close(); @@ -129,8 +138,7 @@ } directoryManager.delete(index.getName()); if (!directoryManager.rename(migrationName, index.getName())) { - throw new IOException("failed to move migrated directory " + - migrationDir); + throw new IOException("failed to move migrated directory " + migrationDir); } log.info("Migrated " + index.getName()); } @@ -150,6 +158,17 @@ this.oldSepChar = oldSepChar; } + @Override + public IndexReader[] getSequentialSubReaders() { + return null; + } + + @Override + public FieldInfos getFieldInfos() { + return ReaderUtil.getMergedFieldInfos(in); + } + + @Override public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException { Document doc = super.document(n, fieldSelector); @@ -167,12 +186,10 @@ return doc; } + @Override public TermEnum terms() throws IOException { List enums = new ArrayList(); - List fieldNames = new ArrayList(); - for (Object obj : in.getFieldNames(FieldOption.ALL)) { - fieldNames.add((String) obj); - } + List fieldNames = new ArrayList(ReaderUtil.getIndexedFields(in)); Collections.sort(fieldNames); for (String fieldName : fieldNames) { if (fieldName.equals(FieldNames.PROPERTIES)) { @@ -184,6 +201,7 @@ return new MigrationTermEnum(new ChainedTermEnum(enums), oldSepChar); } + @Override public TermPositions termPositions() throws IOException { return new MigrationTermPositions(in.termPositions(), oldSepChar); } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitAnalyzer.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitAnalyzer.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitAnalyzer.java (working copy) @@ -23,7 +23,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.standard.ClassicAnalyzer; import org.apache.lucene.util.Version; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,8 +42,8 @@ private static Logger log = LoggerFactory.getLogger(JackrabbitAnalyzer.class); - private static final Analyzer DEFAULT_ANALYZER = - new StandardAnalyzer(Version.LUCENE_24, Collections.emptySet()); + private static final Analyzer DEFAULT_ANALYZER = new ClassicAnalyzer( + Version.LUCENE_36, Collections.emptySet()); /** * Returns a new instance of the named Lucene {@link Analyzer} class, @@ -74,7 +74,7 @@ Class[] types = constructor.getParameterTypes(); if (types.length == 1 && types[0] == Version.class) { try { - return (Analyzer) constructor.newInstance(Version.LUCENE_24); + return (Analyzer) constructor.newInstance(Version.LUCENE_36); } catch (Exception e) { cause = e; } @@ -132,7 +132,7 @@ * Reader. If the fieldName (property) is configured to have a different * analyzer than the default, this analyzer is used for tokenization */ - public TokenStream tokenStream(String fieldName, Reader reader) { + public final TokenStream tokenStream(String fieldName, Reader reader) { if (indexingConfig != null) { Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName); if (propertyAnalyzer != null) { @@ -143,7 +143,7 @@ } @Override - public TokenStream reusableTokenStream(String fieldName, Reader reader) + public final TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { if (indexingConfig != null) { Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName); Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitQueryParser.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitQueryParser.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitQueryParser.java (working copy) @@ -141,11 +141,19 @@ */ protected Query getFieldQuery(String field, String queryText) throws ParseException { + return getFieldQuery(field, queryText, true); + } + + /** + * {@inheritDoc} + */ + protected Query getFieldQuery(String field, String queryText, boolean quoted) + throws ParseException { if (queryText.startsWith("~")) { // synonym query return getSynonymQuery(field, queryText.substring(1)); } else { - return super.getFieldQuery(field, queryText); + return super.getFieldQuery(field, queryText, quoted); } } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitTermQuery.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitTermQuery.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitTermQuery.java (working copy) @@ -40,7 +40,11 @@ } public Weight createWeight(Searcher searcher) throws IOException { - return new JackrabbitTermWeight(searcher, super.createWeight(searcher)); + // use a FilterSearcher to prevent per segment searches + // done by lucene. we handle that on our own + // see instanceof check in lucene TermWeight constructor + return new JackrabbitTermWeight(searcher, + super.createWeight(new FilterSearcher(searcher))); } /** Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryHits.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryHits.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryHits.java (working copy) @@ -49,7 +49,7 @@ this.reader = reader; // We rely on Scorer#nextDoc() and Scorer#advance(int) so enable // scoreDocsInOrder - this.scorer = query.weight(searcher).scorer(reader, true, false); + this.scorer = query.createWeight(searcher).scorer(reader, true, false); } /** Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MoreLikeThis.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MoreLikeThis.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MoreLikeThis.java (working copy) @@ -17,6 +17,7 @@ package org.apache.jackrabbit.core.query.lucene; import org.apache.lucene.util.PriorityQueue; +import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.Version; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; @@ -154,7 +155,7 @@ * Default analyzer to parse source doc with. * @see #getAnalyzer */ - public static final Analyzer DEFAULT_ANALYZER = new StandardAnalyzer(Version.LUCENE_24); + public static final Analyzer DEFAULT_ANALYZER = new StandardAnalyzer(Version.LUCENE_36); /** * Ignore terms with less than this frequency in the source doc. @@ -506,7 +507,7 @@ public Query like(int docNum) throws IOException { if (fieldNames == null) { // gather list of valid fields from lucene - Collection fields = ir.getFieldNames(IndexReader.FieldOption.INDEXED); + Collection fields = ReaderUtil.getIndexedFields(ir); fieldNames = fields.toArray(new String[fields.size()]); } @@ -521,7 +522,7 @@ public Query like(File f) throws IOException { if (fieldNames == null) { // gather list of valid fields from lucene - Collection fields = ir.getFieldNames(IndexReader.FieldOption.INDEXED); + Collection fields = ReaderUtil.getIndexedFields(ir); fieldNames = fields.toArray(new String[fields.size()]); } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiIndex.java (working copy) @@ -287,7 +287,6 @@ handler.getTextAnalyzer(), handler.getSimilarity(), cache, indexingQueue, directoryManager, handler.getMaxHistoryAge()); - index.setMaxFieldLength(handler.getMaxFieldLength()); index.setUseCompoundFile(handler.getUseCompoundFile()); index.setTermInfosIndexDivisor(handler.getTermInfosIndexDivisor()); indexes.add(index); @@ -601,7 +600,6 @@ } throw e; } - index.setMaxFieldLength(handler.getMaxFieldLength()); index.setUseCompoundFile(handler.getUseCompoundFile()); index.setTermInfosIndexDivisor(handler.getTermInfosIndexDivisor()); @@ -1108,7 +1106,6 @@ volatileIndex = new VolatileIndex(handler.getTextAnalyzer(), handler.getSimilarity(), indexingQueue); volatileIndex.setUseCompoundFile(handler.getUseCompoundFile()); - volatileIndex.setMaxFieldLength(handler.getMaxFieldLength()); volatileIndex.setBufferSize(handler.getBufferSize()); } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiScorer.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiScorer.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/MultiScorer.java (working copy) @@ -65,7 +65,7 @@ @Override public int nextDoc() throws IOException { while (currentDoc != NO_MORE_DOCS) { - if (scorers[currentScorer].nextDoc() != NO_MORE_DOCS) { + if (scorers[currentScorer] != null && scorers[currentScorer].nextDoc() != NO_MORE_DOCS) { currentDoc = scorers[currentScorer].docID() + starts[currentScorer]; return currentDoc; } else if (++currentScorer < scorers.length) { Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NodeIndexer.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NodeIndexer.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NodeIndexer.java (working copy) @@ -44,6 +44,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.FieldInfo; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.Parser; import org.slf4j.Logger; @@ -220,7 +221,7 @@ Field parent = new Field(FieldNames.PARENT, false, "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO); - parent.setOmitTermFreqAndPositions(true); + parent.setIndexOptions(FieldInfo.IndexOptions.DOCS_ONLY); doc.add(parent); addNodeName(doc, "", ""); } else if (node.getSharedSet().isEmpty()) { @@ -999,7 +1000,7 @@ Field parentField = new Field(FieldNames.PARENT, false, parentId.toString(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO); - parentField.setOmitTermFreqAndPositions(true); + parentField.setIndexOptions(FieldInfo.IndexOptions.DOCS_ONLY); doc.add(parentField); NodeState parent = (NodeState) stateProvider.getItemState(parentId); ChildNodeEntry child = parent.getChildNodeEntry(node.getNodeId()); Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NotQuery.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NotQuery.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/NotQuery.java (working copy) @@ -139,6 +139,12 @@ public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { contextScorer = context.weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + if (contextScorer == null) { + // context query does not match any node + // the inverse is to match all nodes + return new MatchAllDocsQuery().createWeight(searcher).scorer( + reader, scoreDocsInOrder, false); + } return new NotQueryScorer(reader); } @@ -185,10 +191,6 @@ if (docNo == NO_MORE_DOCS) { return docNo; } - if (contextScorer == null) { - docNo = NO_MORE_DOCS; - return docNo; - } if (docNo == -1) { // get first doc of context scorer @@ -229,10 +231,6 @@ if (docNo == NO_MORE_DOCS) { return docNo; } - if (contextScorer == null) { - docNo = NO_MORE_DOCS; - return docNo; - } // optimize in the case of an advance to finish. // see https://issues.apache.org/jira/browse/JCR-3091 Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ParentAxisQuery.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ParentAxisQuery.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ParentAxisQuery.java (working copy) @@ -190,7 +190,7 @@ */ public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - contextScorer = contextQuery.weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + contextScorer = contextQuery.weight(searcher).scorer(reader, scoreDocsInOrder, false); HierarchyResolver resolver = (HierarchyResolver) reader; return new ParentAxisScorer(searcher.getSimilarity(), reader, searcher, resolver); Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/PredicateDerefQuery.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/PredicateDerefQuery.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/PredicateDerefQuery.java (working copy) @@ -202,9 +202,9 @@ */ public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - subQueryScorer = subQuery.weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + subQueryScorer = subQuery.weight(searcher).scorer(reader, scoreDocsInOrder, false); if (nameTest != null) { - nameTestScorer = new NameQuery(nameTest, version, nsMappings).weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + nameTestScorer = new NameQuery(nameTest, version, nsMappings).weight(searcher).scorer(reader, scoreDocsInOrder, false); } return new DerefScorer(searcher.getSimilarity(), reader); } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java (working copy) @@ -16,9 +16,12 @@ */ package org.apache.jackrabbit.core.query.lucene; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.TermPositions; +import org.apache.lucene.util.ReaderUtil; import java.io.IOException; import java.util.BitSet; @@ -176,7 +179,12 @@ * @exception UnsupportedOperationException always */ @Override - protected void doCommit(Map commitUserData) throws IOException { + protected void doCommit(Map commitUserData) throws IOException { + if (!hasChanges) { + // change in behavior: IndexReader does not check for hasChanges + // before calling doCommit(); + return; + } throw new UnsupportedOperationException("IndexReader is read-only"); } @@ -222,6 +230,16 @@ return new FilteredTermPositions(super.termPositions()); } + @Override + public String toString() { + final StringBuilder buffer = new StringBuilder("ReadOnlyIndexReader("); + buffer.append(in); + buffer.append(','); + buffer.append(deletedDocsVersion); + buffer.append(')'); + return buffer.toString(); + } + //----------------------< FilteredTermDocs >-------------------------------- /** Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/RefCountingIndexReader.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/RefCountingIndexReader.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/RefCountingIndexReader.java (working copy) @@ -16,8 +16,10 @@ */ package org.apache.jackrabbit.core.query.lucene; +import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FilterIndexReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.ReaderUtil; import java.io.IOException; @@ -48,7 +50,7 @@ /** * @return the current reference count value. */ - public synchronized int getRefCount() { + public synchronized int getRefCountJr() { return refCount; } @@ -65,6 +67,16 @@ //-----------------------< FilterIndexReader >-------------------------- + @Override + public IndexReader[] getSequentialSubReaders() { + return null; + } + + @Override + public FieldInfos getFieldInfos() { + return ReaderUtil.getMergedFieldInfos(in); + } + protected void doClose() throws IOException { Util.closeOrRelease(in); } Index: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java =================================================================== --- jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java (revision 1357483) +++ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SearchIndex.java (working copy) @@ -82,6 +82,7 @@ import org.apache.jackrabbit.spi.commons.query.DefaultQueryNodeFactory; import org.apache.jackrabbit.spi.commons.query.qom.OrderingImpl; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.LimitTokenCountAnalyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.analysis.tokenattributes.TermAttribute; @@ -898,7 +899,7 @@ * @return the analyzer in use for indexing. */ public Analyzer getTextAnalyzer() { - return analyzer; + return new LimitTokenCountAnalyzer(analyzer, getMaxFieldLength()); } /** @@ -1743,22 +1744,9 @@ */ private final CachingMultiIndexReader[] subReaders; - /** - * Doc number starts for each sub reader - */ - private int[] starts; - public CombinedIndexReader(CachingMultiIndexReader[] indexReaders) { super(indexReaders); this.subReaders = indexReaders; - this.starts = new int[subReaders.length + 1]; - - int maxDoc = 0; - for (int i = 0; i < subReaders.length; i++) { - starts[i] = maxDoc; - maxDoc += subReaders[i].maxDoc(); - } - starts[subReaders.length] = maxDoc; } /** @@ -1791,36 +1779,6 @@ } } - //---------------------------< internal >------------------------------- - - /** - * Returns the reader index for document n. - * Implementation copied from lucene MultiReader class. - * - * @param n document number. - * @return the reader index. - */ - private int readerIndex(int n) { - int lo = 0; // search starts array - int hi = subReaders.length - 1; // for first element less - - while (hi >= lo) { - int mid = (lo + hi) >> 1; - int midValue = starts[mid]; - if (n < midValue) { - hi = mid - 1; - } else if (n > midValue) { - lo = mid + 1; - } else { // found a match - while (mid + 1 < subReaders.length && starts[mid + 1] == midValue) { - mid++; // scan to last match - } - return mid; - } - } - return hi; - } - public boolean equals(Object obj) { if (obj instanceof CombinedIndexReader) { CombinedIndexReader other = (CombinedIndexReader) obj; Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/AbstractQueryTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/AbstractQueryTest.java (revision 1357483) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/AbstractQueryTest.java (working copy) @@ -214,7 +214,7 @@ // check if all expected are in result for (Iterator it = expectedPaths.iterator(); it.hasNext();) { String path = it.next(); - assertTrue(path + " is not part of the result set "+ expectedPaths, resultPaths.contains(path)); + assertTrue(path + " is not part of the result set "+ resultPaths, resultPaths.contains(path)); } // check result does not contain more than expected for (Iterator it = resultPaths.iterator(); it.hasNext();) { @@ -251,6 +251,7 @@ */ protected QueryResult executeQuery(String statement) throws RepositoryException { + getSearchIndex().flush(); if (statement.trim().toLowerCase().startsWith("select")) { return qm.createQuery(statement, Query.SQL).execute(); } else { Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/ChildAxisQueryTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/ChildAxisQueryTest.java (revision 1357483) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/ChildAxisQueryTest.java (working copy) @@ -253,6 +253,6 @@ + testRootNode.getPath() + "') and not isdescendantnode(a,'" + foo.getPath() + "')"; - executeSQL2Query(sql, new Node[] {}); + executeSQL2Query(sql, new Node[] {foo}); } } Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/lucene/ChainedTermEnumTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/lucene/ChainedTermEnumTest.java (revision 1357483) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/lucene/ChainedTermEnumTest.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermEnum; import org.apache.lucene.store.Directory; @@ -61,21 +62,28 @@ protected TermEnum createTermEnum(String prefix, int numTerms) throws IOException { Directory dir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(Version.LUCENE_24), - true, IndexWriter.MaxFieldLength.UNLIMITED); - for (int i = 0; i < numTerms; i++) { - Document doc = new Document(); - doc.add(new Field("field", true, prefix + i, Field.Store.NO, - Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO)); - writer.addDocument(doc); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( + Version.LUCENE_36, new StandardAnalyzer(Version.LUCENE_36))); + try { + for (int i = 0; i < numTerms; i++) { + Document doc = new Document(); + doc.add(new Field("field", true, prefix + i, Field.Store.NO, + Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO)); + writer.addDocument(doc); + } + } finally { + writer.close(); } - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - TermEnum terms = reader.terms(); - if (terms.term() == null) { - // position at first term - terms.next(); + IndexReader reader = IndexReader.open(dir); + try { + TermEnum terms = reader.terms(); + if (terms.term() == null) { + // position at first term + terms.next(); + } + return terms; + } finally { + reader.close(); } - return terms; } } Index: jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/lucene/IndexMigrationTest.java =================================================================== --- jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/lucene/IndexMigrationTest.java (revision 1357483) +++ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/query/lucene/IndexMigrationTest.java (working copy) @@ -50,7 +50,7 @@ DirectoryManager dirMgr = new RAMDirectoryManager(); PersistentIndex idx = new PersistentIndex("index", - new StandardAnalyzer(Version.LUCENE_24), Similarity.getDefault(), + new StandardAnalyzer(Version.LUCENE_36), Similarity.getDefault(), new DocNumberCache(100), new IndexingQueue(new IndexingQueueStore(new RAMDirectory())), dirMgr, 0); Index: jackrabbit-parent/pom.xml =================================================================== --- jackrabbit-parent/pom.xml (revision 1357483) +++ jackrabbit-parent/pom.xml (working copy) @@ -362,7 +362,7 @@ org.apache.lucene lucene-core - 3.0.3 + 3.6.0 org.apache.tika