Index: src/main/java/org/apache/jackrabbit/core/query/lucene/PersistentIndex.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/PersistentIndex.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/PersistentIndex.java (working copy) @@ -112,6 +112,7 @@ */ void addIndexes(IndexReader[] readers) throws IOException { getIndexWriter().addIndexes(readers); + //getIndexWriter().forceMerge(1, true); getIndexWriter().optimize(); } Index: src/main/java/org/apache/jackrabbit/core/query/lucene/VolatileIndex.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/VolatileIndex.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/VolatileIndex.java (working copy) @@ -105,7 +105,7 @@ num = 1; } else { // remove document from index - num = super.getIndexReader().deleteDocuments(idTerm); + num = super.removeDocument(idTerm); } numDocs -= num; return num; Index: src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitQueryParser.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitQueryParser.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitQueryParser.java (working copy) @@ -56,7 +56,7 @@ Analyzer analyzer, SynonymProvider synonymProvider, PerQueryCache cache) { - super(Version.LUCENE_24, fieldName, analyzer); + super(Version.LUCENE_35, fieldName, analyzer); this.synonymProvider = synonymProvider; this.cache = cache; setAllowLeadingWildcard(true); @@ -135,17 +135,24 @@ } } - /** * {@inheritDoc} */ protected Query getFieldQuery(String field, String queryText) throws ParseException { + return getFieldQuery(field, queryText, true); + } + + /** + * {@inheritDoc} + */ + protected Query getFieldQuery(String field, String queryText, boolean quoted) + throws ParseException { if (queryText.startsWith("~")) { // synonym query return getSynonymQuery(field, queryText.substring(1)); } else { - return super.getFieldQuery(field, queryText); + return super.getFieldQuery(field, queryText, quoted); } } Index: src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractIndex.java (working copy) @@ -36,10 +36,13 @@ import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Payload; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Version; import org.apache.tika.io.IOExceptionWithCause; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,11 +87,11 @@ /** The similarity in use for indexing and searching. */ private final Similarity similarity; - /** Compound file flag */ - private boolean useCompoundFile = true; - - /** maxFieldLength config parameter */ - private int maxFieldLength = SearchIndex.DEFAULT_MAX_FIELD_LENGTH; +// /** Compound file flag */ +// private boolean useCompoundFile = true; +// +// /** maxFieldLength config parameter */ +// private int maxFieldLength = SearchIndex.DEFAULT_MAX_FIELD_LENGTH; /** termInfosIndexDivisor config parameter */ private int termInfosIndexDivisor = SearchIndex.DEFAULT_TERM_INFOS_INDEX_DIVISOR; @@ -143,8 +146,7 @@ this.isExisting = IndexReader.indexExists(directory); if (!isExisting) { - indexWriter = new IndexWriter(directory, analyzer, - IndexWriter.MaxFieldLength.LIMITED); + indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, analyzer)); // immediately close, now that index has been created indexWriter.close(); indexWriter = null; @@ -352,10 +354,11 @@ indexReader = null; } if (indexWriter == null) { - indexWriter = new IndexWriter(getDirectory(), analyzer, - new IndexWriter.MaxFieldLength(maxFieldLength)); - indexWriter.setSimilarity(similarity); - indexWriter.setUseCompoundFile(useCompoundFile); + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_35, analyzer); + config.setSimilarity(similarity); + + indexWriter = new IndexWriter(getDirectory(), config); +// indexWriter.setUseCompoundFile(useCompoundFile); indexWriter.setInfoStream(STREAM_LOGGER); } return indexWriter; @@ -388,6 +391,7 @@ // optimize if requested if (optimize) { IndexWriter writer = getIndexWriter(); + //writer.forceMerge(1, true); writer.optimize(); writer.close(); indexWriter = null; @@ -536,22 +540,27 @@ /** * The lucene index writer property: useCompountFile + * + * @deprecated lucene now uses {@link TieredMergePolicy} set via the {@link IndexWriterConfig} */ + @Deprecated void setUseCompoundFile(boolean b) { - useCompoundFile = b; - if (indexWriter != null) { - indexWriter.setUseCompoundFile(b); - } +// useCompoundFile = b; +// if (indexWriter != null) { +// indexWriter.setUseCompoundFile(b); +// } } /** * The lucene index writer property: maxFieldLength + * + * @deprecated */ void setMaxFieldLength(int maxFieldLength) { - this.maxFieldLength = maxFieldLength; - if (indexWriter != null) { - indexWriter.setMaxFieldLength(maxFieldLength); - } +// this.maxFieldLength = maxFieldLength; +// if (indexWriter != null) { +// indexWriter.setMaxFieldLength(maxFieldLength); +// } } /** Index: src/main/java/org/apache/jackrabbit/core/query/lucene/directory/FSDirectoryManager.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/directory/FSDirectoryManager.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/directory/FSDirectoryManager.java (working copy) @@ -152,7 +152,7 @@ @Override public String[] listAll() throws IOException { - File[] files = directory.getFile().listFiles(FILTER); + File[] files = directory.getDirectory().listFiles(FILTER); if (files == null) { return null; } @@ -173,7 +173,12 @@ return directory.fileModified(name); } + /** + * @deprecated Lucene will remove this method + * @see org.apache.lucene.store.Directory#touchFile(java.lang.String) + */ @Override + @Deprecated public void touchFile(String name) throws IOException { directory.touchFile(name); } @@ -196,7 +201,7 @@ @Override public IndexInput openInput(String name) throws IOException { IndexInput in = directory.openInput(name); - return new IndexInputLogWrapper(in); + return new IndexInputLogWrapper(name, in); } @Override @@ -208,7 +213,7 @@ public IndexInput openInput(String name, int bufferSize) throws IOException { IndexInput in = directory.openInput(name, bufferSize); - return new IndexInputLogWrapper(in); + return new IndexInputLogWrapper(name, in); } @Override @@ -222,7 +227,7 @@ } @Override - public void setLockFactory(LockFactory lockFactory) { + public void setLockFactory(LockFactory lockFactory) throws IOException { directory.setLockFactory(lockFactory); } @@ -249,7 +254,8 @@ private IndexInput in; - IndexInputLogWrapper(IndexInput in) { + IndexInputLogWrapper(String name, IndexInput in) { + super(name); this.in = in; } Index: src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/ReadOnlyIndexReader.java (working copy) @@ -171,7 +171,13 @@ * @exception UnsupportedOperationException always */ @Override - protected void doCommit(Map commitUserData) throws IOException { + protected void doCommit(Map commitUserData) + throws IOException { + if (!hasChanges) { + // change in behavior: IndexReader does not check for hasChanges + // before calling doCommit(); + return; + } throw new UnsupportedOperationException("IndexReader is read-only"); } Index: src/main/java/org/apache/jackrabbit/core/query/lucene/MoreLikeThis.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/MoreLikeThis.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/MoreLikeThis.java (working copy) @@ -154,7 +154,7 @@ * Default analyzer to parse source doc with. * @see #getAnalyzer */ - public static final Analyzer DEFAULT_ANALYZER = new StandardAnalyzer(Version.LUCENE_24); + public static final Analyzer DEFAULT_ANALYZER = new StandardAnalyzer(Version.LUCENE_35); /** * Ignore terms with less than this frequency in the source doc. Index: src/main/java/org/apache/jackrabbit/core/query/lucene/IndexMigration.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/IndexMigration.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/IndexMigration.java (working copy) @@ -36,10 +36,12 @@ import org.apache.lucene.index.FilterIndexReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermEnum; import org.apache.lucene.index.TermPositions; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Version; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,15 +104,17 @@ // if we get here then the index must be migrated log.debug("Index requires migration {}", indexDir); - String migrationName = index.getName() + "_v2.3"; + String migrationName = index.getName() + "_v35"; if (directoryManager.hasDirectory(migrationName)) { directoryManager.delete(migrationName); } Directory migrationDir = directoryManager.getDirectory(migrationName); + // try { - IndexWriter writer = new IndexWriter(migrationDir, new JackrabbitAnalyzer(), - IndexWriter.MaxFieldLength.UNLIMITED); + IndexWriter writer = new IndexWriter(migrationDir, + new IndexWriterConfig(Version.LUCENE_35, + new JackrabbitAnalyzer())); try { IndexReader r = new MigrationIndexReader( IndexReader.open(index.getDirectory(), true), Index: src/test/java/org/apache/jackrabbit/core/query/FulltextQueryTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/core/query/FulltextQueryTest.java (revision 1300043) +++ src/test/java/org/apache/jackrabbit/core/query/FulltextQueryTest.java (working copy) @@ -19,7 +19,6 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.Callable; import javax.jcr.Node; import javax.jcr.RepositoryException; @@ -373,6 +372,7 @@ stmt.append("/jcr:root").append(testRoot).append("/*"); stmt.append("[jcr:contains(., '").append(statement); stmt.append("')]"); + getSearchIndex().flush(); Query q = superuser.getWorkspace().getQueryManager().createQuery(stmt.toString(), Query.XPATH); checkResult(q.execute(), match ? 1 : 0); Index: src/main/java/org/apache/jackrabbit/core/query/lucene/IDField.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/IDField.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/IDField.java (working copy) @@ -21,6 +21,8 @@ import org.apache.jackrabbit.core.id.NodeId; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.AbstractField; +import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.FieldInfo.IndexOptions; /** * IDField implements a lucene field for the id of a node. @@ -37,7 +39,8 @@ this.isStored = true; this.isTokenized = false; this.omitNorms = true; - this.omitTermFreqAndPositions = true; + setIndexOptions(IndexOptions.DOCS_ONLY); + setStoreTermVector(TermVector.NO); } public String stringValue() { Index: src/main/java/org/apache/jackrabbit/core/query/lucene/DescendantSelfAxisQuery.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/DescendantSelfAxisQuery.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/DescendantSelfAxisQuery.java (working copy) @@ -392,8 +392,10 @@ */ public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - contextScorer = contextQuery.weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); - subScorer = subQuery.weight(searcher).scorer(reader, scoreDocsInOrder, topScorer); + contextScorer = searcher.createNormalizedWeight(contextQuery) + .scorer(reader, scoreDocsInOrder, topScorer); + subScorer = searcher.createNormalizedWeight(subQuery).scorer( + reader, scoreDocsInOrder, topScorer); HierarchyResolver resolver = (HierarchyResolver) reader; return new DescendantSelfAxisScorer(searcher.getSimilarity(), reader, resolver); } @@ -471,9 +473,14 @@ } collectContextHits(); - currentDoc = subScorer.nextDoc(); if (contextHits.isEmpty()) { currentDoc = NO_MORE_DOCS; + } else { + if (subScorer != null) { + currentDoc = subScorer.nextDoc(); + } else { + currentDoc = NO_MORE_DOCS; + } } while (currentDoc != NO_MORE_DOCS) { if (isValid(currentDoc)) { @@ -505,7 +512,9 @@ // optimize in the case of an advance to finish. // see https://issues.apache.org/jira/browse/JCR-3082 if (target == NO_MORE_DOCS) { - subScorer.advance(target); + if (subScorer != null) { + subScorer.advance(target); + } currentDoc = NO_MORE_DOCS; return currentDoc; } Index: src/main/java/org/apache/jackrabbit/core/query/lucene/SortedLuceneQueryHits.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/SortedLuceneQueryHits.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/SortedLuceneQueryHits.java (working copy) @@ -152,7 +152,7 @@ //-------------------------------< internal >------------------------------- private void getHits() throws IOException { - TopFieldCollector collector = TopFieldCollector.create(sort, numHits, false, true, false, false); + TopFieldCollector collector = TopFieldCollector.create(sort, numHits, false, true, false, true); searcher.search(query, collector); size = collector.getTotalHits(); offset += scoreDocs.length; Index: src/test/java/org/apache/jackrabbit/core/query/lucene/IndexMigrationTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/core/query/lucene/IndexMigrationTest.java (revision 1300043) +++ src/test/java/org/apache/jackrabbit/core/query/lucene/IndexMigrationTest.java (working copy) @@ -17,6 +17,8 @@ package org.apache.jackrabbit.core.query.lucene; import junit.framework.TestCase; + +import org.apache.jackrabbit.core.id.NodeId; import org.apache.jackrabbit.core.query.lucene.directory.DirectoryManager; import org.apache.jackrabbit.core.query.lucene.directory.RAMDirectoryManager; import org.apache.lucene.analysis.standard.StandardAnalyzer; @@ -41,6 +43,7 @@ private static final char SEP_CHAR = '\uFFFE'; public void testMigration() throws Exception { + List docs = new ArrayList(); docs.add(createDocument("ab", "a")); docs.add(createDocument("a", "b")); @@ -50,7 +53,7 @@ DirectoryManager dirMgr = new RAMDirectoryManager(); PersistentIndex idx = new PersistentIndex("index", - new StandardAnalyzer(Version.LUCENE_24), Similarity.getDefault(), + new StandardAnalyzer(Version.LUCENE_35), Similarity.getDefault(), new DocNumberCache(100), new IndexingQueue(new IndexingQueueStore(new RAMDirectory())), dirMgr, 0); @@ -66,8 +69,8 @@ protected static Document createDocument(String name, String value) { Document doc = new Document(); - doc.add(new Field(FieldNames.UUID, false, UUID.randomUUID().toString(), - Field.Store.YES, Field.Index.NO, Field.TermVector.NO)); + // id first + doc.add(new IDField(NodeId.valueOf(UUID.randomUUID().toString()))); doc.add(new Field(FieldNames.PROPERTIES, false, createNamedValue14( name, value), Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.NO)); Index: src/test/java/org/apache/jackrabbit/core/query/lucene/ChainedTermEnumTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/core/query/lucene/ChainedTermEnumTest.java (revision 1300043) +++ src/test/java/org/apache/jackrabbit/core/query/lucene/ChainedTermEnumTest.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermEnum; import org.apache.lucene.store.Directory; @@ -61,8 +62,8 @@ protected TermEnum createTermEnum(String prefix, int numTerms) throws IOException { Directory dir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(Version.LUCENE_24), - true, IndexWriter.MaxFieldLength.UNLIMITED); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( + Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35))); for (int i = 0; i < numTerms; i++) { Document doc = new Document(); doc.add(new Field("field", true, prefix + i, Field.Store.NO, Index: src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryHits.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryHits.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/LuceneQueryHits.java (working copy) @@ -49,7 +49,7 @@ this.reader = reader; // We rely on Scorer#nextDoc() and Scorer#advance(int) so enable // scoreDocsInOrder - this.scorer = query.weight(searcher).scorer(reader, true, false); + this.scorer = query.createWeight(searcher).scorer(reader, true, false); } /** Index: src/test/java/org/apache/jackrabbit/core/TestHelper.java =================================================================== --- src/test/java/org/apache/jackrabbit/core/TestHelper.java (revision 1292734) +++ src/test/java/org/apache/jackrabbit/core/TestHelper.java (working copy) @@ -105,13 +105,17 @@ /** * wait for async text-extraction tasks to finish */ - public static void waitForTextExtractionTasksToFinish(Session session) throws Exception { + public static void waitForTextExtractionTasksToFinish(Session session) { final RepositoryContext context = JackrabbitRepositoryStub .getRepositoryContext(session.getRepository()); JackrabbitThreadPool jtp = ((JackrabbitThreadPool) context .getExecutor()); while (jtp.getPendingLowPriorityTaskCount() != 0) { - TimeUnit.MILLISECONDS.sleep(100); + try { + TimeUnit.MILLISECONDS.sleep(100); + } catch (InterruptedException e) { + // + } } } } Index: src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitAnalyzer.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitAnalyzer.java (revision 1300043) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/JackrabbitAnalyzer.java (working copy) @@ -23,7 +23,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.standard.ClassicAnalyzer; import org.apache.lucene.util.Version; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,8 +42,8 @@ private static Logger log = LoggerFactory.getLogger(JackrabbitAnalyzer.class); - private static final Analyzer DEFAULT_ANALYZER = - new StandardAnalyzer(Version.LUCENE_24, Collections.emptySet()); + private static final Analyzer DEFAULT_ANALYZER = new ClassicAnalyzer( + Version.LUCENE_35, Collections.emptySet()); /** * Returns a new instance of the named Lucene {@link Analyzer} class, @@ -132,7 +132,7 @@ * Reader. If the fieldName (property) is configured to have a different * analyzer than the default, this analyzer is used for tokenization */ - public TokenStream tokenStream(String fieldName, Reader reader) { + public final TokenStream tokenStream(String fieldName, Reader reader) { if (indexingConfig != null) { Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName); if (propertyAnalyzer != null) { @@ -143,7 +143,7 @@ } @Override - public TokenStream reusableTokenStream(String fieldName, Reader reader) + public final TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { if (indexingConfig != null) { Analyzer propertyAnalyzer = indexingConfig.getPropertyAnalyzer(fieldName);