Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java (revision 1716068) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java (working copy) @@ -134,7 +134,7 @@ } builder.append(v); } - return new OakTextField(FieldNames.SUGGEST, builder.toString(), true); + return new OakTextField(FieldNames.SUGGEST, builder.toString(), false); } /** Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java (revision 1716068) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java (working copy) @@ -30,24 +30,26 @@ import javax.annotation.Nullable; import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.SuggestHelper; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; class IndexNode { - static IndexNode open(String indexPath, NodeState root, NodeState defnNodeState,@Nullable IndexCopier cloner) + static IndexNode open(String indexPath, NodeState root, NodeState defnNodeState, @Nullable IndexCopier cloner) throws IOException { Directory directory = null; IndexDefinition definition = new IndexDefinition(root, defnNodeState, indexPath); NodeState data = defnNodeState.getChildNode(INDEX_DATA_CHILD_NAME); if (data.exists()) { directory = new OakDirectory(new ReadOnlyBuilder(defnNodeState), definition, true); - if (cloner != null){ + if (cloner != null) { directory = cloner.wrapForRead(indexPath, definition, directory); } } else if (PERSISTENCE_FILE.equalsIgnoreCase(defnNodeState.getString(PERSISTENCE_NAME))) { @@ -59,7 +61,12 @@ if (directory != null) { try { - IndexNode index = new IndexNode(PathUtils.getName(indexPath), definition, directory); + OakDirectory suggestDirectory = null; + if (definition.isSuggestEnabled()) { + suggestDirectory = new OakDirectory(defnNodeState.builder(), ":suggest-data", definition, false); + } + + IndexNode index = new IndexNode(PathUtils.getName(indexPath), definition, directory, suggestDirectory); directory = null; // closed in Index.close() return index; } finally { @@ -84,15 +91,22 @@ private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final AnalyzingInfixSuggester lookup; + private boolean closed = false; - IndexNode(String name, IndexDefinition definition, Directory directory) + IndexNode(String name, IndexDefinition definition, Directory directory, final OakDirectory suggestDirectory) throws IOException { this.name = name; this.definition = definition; this.directory = directory; this.reader = DirectoryReader.open(directory); this.searcher = new IndexSearcher(reader); + if (suggestDirectory != null) { + this.lookup = SuggestHelper.getLookup(suggestDirectory, definition.getAnalyzer()); + } else { + this.lookup = null; + } } String getName() { @@ -107,6 +121,10 @@ return searcher; } + AnalyzingInfixSuggester getLookup() { + return lookup; + } + boolean acquire() { lock.readLock().lock(); if (closed) { Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (revision 1716068) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (working copy) @@ -412,7 +412,7 @@ noDocs = true; } else if (luceneRequestFacade.getLuceneRequest() instanceof SuggestHelper.SuggestQuery) { SuggestHelper.SuggestQuery suggestQuery = (SuggestHelper.SuggestQuery) luceneRequestFacade.getLuceneRequest(); - List lookupResults = SuggestHelper.getSuggestions(suggestQuery); + List lookupResults = SuggestHelper.getSuggestions(indexNode.getLookup(), suggestQuery); // ACL filter suggestions Collection suggestedWords = new ArrayList(lookupResults.size()); Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (revision 1716068) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (working copy) @@ -380,7 +380,7 @@ // because of LUCENE-5833 we have to merge the suggest fields into a single one Field suggestField = null; for (Field f : fields) { - if (FieldNames.SUGGEST.endsWith(f.name())) { + if (FieldNames.SUGGEST.equals(f.name())) { if (suggestField == null) { suggestField = FieldFactory.newSuggestField(f.stringValue()); } else { @@ -425,7 +425,7 @@ } if (pd.useInSuggest) { - fields.add(newPropertyField(FieldNames.SUGGEST, value, true, true)); + fields.add(FieldFactory.newSuggestField(value)); } if (pd.useInSpellcheck) { Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (revision 1716068) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (working copy) @@ -79,6 +79,7 @@ Analyzer definitionAnalyzer = definition.getAnalyzer(); Map analyzers = new HashMap(); analyzers.put(FieldNames.SPELLCHECK, new ShingleAnalyzerWrapper(LuceneIndexConstants.ANALYZER, 3)); + analyzers.put(FieldNames.SUGGEST, SuggestHelper.getAnalyzer()); Analyzer analyzer = new PerFieldAnalyzerWrapper(definitionAnalyzer, analyzers); IndexWriterConfig config = new IndexWriterConfig(VERSION, analyzer); if (remoteDir) { @@ -136,7 +137,6 @@ @Nullable private final IndexCopier indexCopier; - private Directory directory; private final TextExtractionStats textExtractionStats = new TextExtractionStats(); @@ -190,12 +190,12 @@ checkNotNull(writer); checkNotNull(definition); checkNotNull(directory); - + int docs = writer.numDocs(); int ram = writer.numRamDocs(); log.trace("Writer for direcory {} - docs: {}, ramDocs: {}", definition, docs, ram); - + String[] files = directory.listAll(); long overallSize = 0; StringBuilder sb = new StringBuilder(); @@ -231,18 +231,18 @@ if (log.isTraceEnabled()) { trackIndexSizeInfo(writer, definition, directory); } - + final long start = PERF_LOGGER.start(); - - updateSuggester(); + + updateSuggester(writer.getAnalyzer()); PERF_LOGGER.end(start, -1, "Completed suggester for directory {}", definition); - + writer.close(); PERF_LOGGER.end(start, -1, "Closed writer for directory {}", definition); - + directory.close(); PERF_LOGGER.end(start, -1, "Closed directory for directory {}", definition); - + //OAK-2029 Record the last updated status so //as to make IndexTracker detect changes when index //is stored in file system @@ -250,7 +250,7 @@ status.setProperty("lastUpdated", ISO8601.format(Calendar.getInstance()), Type.DATE); status.setProperty("indexedNodes",indexedNodes); PERF_LOGGER.end(start, -1, "Overall Closed IndexWriter for directory {}", definition); - + textExtractionStats.log(reindex); textExtractionStats.collectStats(extractedTextCache); } @@ -259,8 +259,9 @@ /** * eventually update suggest dictionary * @throws IOException if suggest dictionary update fails + * @param analyzer the analyzer used to update the suggester */ - private void updateSuggester() throws IOException { + private void updateSuggester(Analyzer analyzer) throws IOException { if (definition.isSuggestEnabled()) { @@ -280,12 +281,14 @@ if (updateSuggester) { DirectoryReader reader = DirectoryReader.open(writer, false); + final OakDirectory suggestDirectory = new OakDirectory(definitionBuilder, ":suggest-data", definition, false); try { - SuggestHelper.updateSuggester(reader); + SuggestHelper.updateSuggester(suggestDirectory, analyzer, reader); suggesterStatus.setProperty("lastUpdated", ISO8601.format(Calendar.getInstance()), Type.DATE); } catch (Throwable e) { log.warn("could not update suggester", e); } finally { + suggestDirectory.close(); reader.close(); } } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (revision 1716068) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (working copy) @@ -32,7 +32,6 @@ import java.util.concurrent.atomic.AtomicReference; import com.google.common.collect.AbstractIterator; -import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Queues; import com.google.common.collect.Sets; @@ -78,6 +77,7 @@ import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Term; import org.apache.lucene.queries.CustomScoreQuery; +import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; @@ -406,13 +406,14 @@ } } } else if (luceneRequestFacade.getLuceneRequest() instanceof SpellcheckHelper.SpellcheckQuery) { + String aclCheckField = indexNode.getDefinition().isFullTextEnabled() ? FieldNames.FULLTEXT : FieldNames.SPELLCHECK; SpellcheckHelper.SpellcheckQuery spellcheckQuery = (SpellcheckHelper.SpellcheckQuery) luceneRequestFacade.getLuceneRequest(); SuggestWord[] suggestWords = SpellcheckHelper.getSpellcheck(spellcheckQuery); // ACL filter spellchecks - QueryParser qp = new QueryParser(Version.LUCENE_47, FieldNames.FULLTEXT, indexNode.getDefinition().getAnalyzer()); + QueryParser qp = new QueryParser(Version.LUCENE_47, aclCheckField, indexNode.getDefinition().getAnalyzer()); for (SuggestWord suggestion : suggestWords) { - Query query = qp.createPhraseQuery(FieldNames.FULLTEXT, suggestion.string); + Query query = qp.createPhraseQuery(aclCheckField, suggestion.string); TopDocs topDocs = searcher.search(query, 100); if (topDocs.totalHits > 0) { for (ScoreDoc doc : topDocs.scoreDocs) { @@ -427,14 +428,14 @@ noDocs = true; } else if (luceneRequestFacade.getLuceneRequest() instanceof SuggestHelper.SuggestQuery) { + QueryParser qp = new MultiFieldQueryParser(Version.LUCENE_47, MultiFields.getIndexedFields(searcher.getIndexReader()).toArray(new String[0]), indexNode.getDefinition().getAnalyzer()); SuggestHelper.SuggestQuery suggestQuery = (SuggestHelper.SuggestQuery) luceneRequestFacade.getLuceneRequest(); - List lookupResults = SuggestHelper.getSuggestions(suggestQuery); + List lookupResults = SuggestHelper.getSuggestions(indexNode.getLookup(), suggestQuery); // ACL filter suggestions - QueryParser qp = new QueryParser(Version.LUCENE_47, FieldNames.SUGGEST, indexNode.getDefinition().getAnalyzer()); for (Lookup.LookupResult suggestion : lookupResults) { - Query query = qp.createPhraseQuery(FieldNames.SUGGEST, suggestion.key.toString()); + Query query = qp.parse("\"" + suggestion.key.toString() + "\""); TopDocs topDocs = searcher.search(query, 100); if (topDocs.totalHits > 0) { for (ScoreDoc doc : topDocs.scoreDocs) { @@ -449,7 +450,7 @@ noDocs = true; } - } catch (IOException e) { + } catch (Exception e) { LOG.warn("query via {} failed.", LucenePropertyIndex.this, e); } finally { indexNode.release(); Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java (revision 1716068) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/OakDirectory.java (working copy) @@ -84,9 +84,13 @@ private final boolean activeDeleteEnabled; public OakDirectory(NodeBuilder builder, IndexDefinition definition, boolean readOnly) { + this(builder, INDEX_DATA_CHILD_NAME, definition, readOnly); + } + + public OakDirectory(NodeBuilder builder, String dataNodeName, IndexDefinition definition, boolean readOnly) { this.lockFactory = NoLockFactory.getNoLockFactory(); this.builder = builder; - this.directoryBuilder = builder.child(INDEX_DATA_CHILD_NAME); + this.directoryBuilder = builder.child(dataNodeName); this.definition = definition; this.readOnly = readOnly; this.fileNames.addAll(getListing()); Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SuggestHelper.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SuggestHelper.java (revision 1716068) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SuggestHelper.java (working copy) @@ -18,17 +18,22 @@ */ package org.apache.jackrabbit.oak.plugins.index.lucene.util; +import java.io.File; import java.io.IOException; import java.io.Reader; import java.util.Collections; import java.util.List; +import com.google.common.io.Files; import org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.suggest.DocumentDictionary; +import org.apache.lucene.search.spell.Dictionary; +import org.apache.lucene.search.spell.LuceneDictionary; import org.apache.lucene.search.suggest.Lookup; -import org.apache.lucene.search.suggest.analyzing.FreeTextSuggester; +import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Version; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,29 +53,20 @@ } }; - private static final Lookup suggester = new FreeTextSuggester(analyzer); - - public static void updateSuggester(IndexReader reader) throws IOException { -// Terms terms = MultiFields.getTerms(reader, FieldNames.SUGGEST); -// long size = terms.size() * 2; -// if (size < 0) { -// size = terms.getDocCount() / 3; -// } -// long count = suggester.getCount(); -// if (size > count) { - try { - suggester.build(new DocumentDictionary(reader, FieldNames.SUGGEST, FieldNames.PATH_DEPTH)); - } catch (RuntimeException e) { - log.debug("could not update the suggester", e); - } -// } + public static void updateSuggester(Directory directory, Analyzer analyzer, IndexReader reader) throws IOException { + try { + Dictionary dictionary = new LuceneDictionary(reader, FieldNames.SUGGEST); + getLookup(directory, analyzer).build(dictionary); + } catch (RuntimeException e) { + log.debug("could not update the suggester", e); + } } - public static List getSuggestions(SuggestQuery suggestQuery) { + public static List getSuggestions(AnalyzingInfixSuggester suggester, SuggestQuery suggestQuery) { try { long count = suggester.getCount(); if (count > 0) { - return suggester.lookup(suggestQuery.getText(), false, 10); + return suggester.lookup(suggestQuery.getText(), 10, true, false); } else { return Collections.emptyList(); } @@ -103,6 +99,28 @@ } } + public static AnalyzingInfixSuggester getLookup(final Directory suggestDirectory) throws IOException { + return getLookup(suggestDirectory, SuggestHelper.analyzer); + } + + public static AnalyzingInfixSuggester getLookup(final Directory suggestDirectory, Analyzer analyzer) throws IOException { + final File tempDir = Files.createTempDir(); + return new AnalyzingInfixSuggester(Version.LUCENE_47, tempDir, analyzer, analyzer, 3) { + @Override + protected Directory getDirectory(File path) throws IOException { + if (tempDir.getAbsolutePath().equals(path.getAbsolutePath())) { + return suggestDirectory; // use oak directory for writing suggest index + } else { + return FSDirectory.open(path); // use FS for temp index used at build time + } + } + }; + } + + public static Analyzer getAnalyzer() { + return analyzer; + } + public static class SuggestQuery { private final String text; Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/package-info.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/package-info.java (revision 1716068) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/package-info.java (working copy) @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@Version("1.2.0") +@Version("2.0.0") @Export(optional = "provide:=true") package org.apache.jackrabbit.oak.plugins.index.lucene.util; Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java (revision 1716068) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/SuggestTest.java (working copy) @@ -90,6 +90,22 @@ assertTrue(result.contains("in 2015 my fox is red, like mike's fox and john's fox")); } + public void testSuggestInfix() throws Exception { + Session session = superuser; + QueryManager qm = session.getWorkspace().getQueryManager(); + Node n1 = testRootNode.addNode("node1"); + n1.setProperty("jcr:title", "in 2015 my fox is red, like mike's fox and john's fox"); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty("jcr:title", "in 2015 a red fox is still a fox"); + session.save(); + + String xpath = "/jcr:root[rep:suggest('like mike')]/(rep:suggest())"; + Query q = qm.createQuery(xpath, Query.XPATH); + List result = getResult(q.execute(), "rep:suggest()"); + assertNotNull(result); + assertTrue(result.contains("in 2015 my fox is red, like mike's fox and john's fox")); + } + public void testNoSuggestions() throws Exception { Session session = superuser; QueryManager qm = session.getWorkspace().getQueryManager(); Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java (revision 1716068) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java (working copy) @@ -460,11 +460,11 @@ } private IndexNode createIndexNode(IndexDefinition defn, long numOfDocs) throws IOException { - return new IndexNode("foo", defn, createSampleDirectory(numOfDocs)); + return new IndexNode("foo", defn, createSampleDirectory(numOfDocs), null); } private IndexNode createIndexNode(IndexDefinition defn) throws IOException { - return new IndexNode("foo", defn, createSampleDirectory()); + return new IndexNode("foo", defn, createSampleDirectory(), null); } private FilterImpl createFilter(String nodeTypeName) {