diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java index 0d5a445..7b079ab 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java @@ -57,6 +57,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.util.TreeUtil; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer; import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory; import org.apache.lucene.codecs.Codec; @@ -116,6 +117,11 @@ class IndexDefinition implements Aggregate.AggregateMapper{ static final long DEFAULT_ENTRY_COUNT = 1000; /** + * Default value for property {@link #maxFieldLength}. + */ + public static final int DEFAULT_MAX_FIELD_LENGTH = 10000; + + /** * System managed hidden property to record the current index version */ static final String INDEX_VERSION = ":version"; @@ -181,6 +187,8 @@ class IndexDefinition implements Aggregate.AggregateMapper{ private final boolean hasCustomTikaConfig; + private final int maxFieldLength; + public IndexDefinition(NodeState root, NodeState defn) { this(root, defn, null); } @@ -231,6 +239,7 @@ class IndexDefinition implements Aggregate.AggregateMapper{ this.entryCount = DEFAULT_ENTRY_COUNT; } + this.maxFieldLength = getOptionalValue(defn, LuceneIndexConstants.MAX_FIELD_LENGTH, DEFAULT_MAX_FIELD_LENGTH); this.costPerEntry = getOptionalValue(defn, LuceneIndexConstants.COST_PER_ENTRY, 1.0); this.costPerExecution = getOptionalValue(defn, LuceneIndexConstants.COST_PER_EXECUTION, 1.0); this.indexesAllTypes = areAllTypesIndexed(); @@ -327,24 +336,27 @@ class IndexDefinition implements Aggregate.AggregateMapper{ @Override public String toString() { - return "IndexDefinition : " + indexName; + return "Lucene Index : " + indexName; } //~---------------------------------------------------< Analyzer > private Analyzer createAnalyzer() { + Analyzer result = null; Analyzer defaultAnalyzer = LuceneIndexConstants.ANALYZER; if (analyzers.containsKey(LuceneIndexConstants.ANL_DEFAULT)){ defaultAnalyzer = analyzers.get(LuceneIndexConstants.ANL_DEFAULT); } if (!evaluatePathRestrictions()){ - return defaultAnalyzer; + result = defaultAnalyzer; + } else { + Map analyzerMap = ImmutableMap.builder() + .put(FieldNames.ANCESTORS, + new TokenizerChain(new PathHierarchyTokenizerFactory(Collections.emptyMap()))) + .build(); + result = new PerFieldAnalyzerWrapper(defaultAnalyzer, analyzerMap); } - Map analyzerMap = ImmutableMap.builder() - .put(FieldNames.ANCESTORS, - new TokenizerChain(new PathHierarchyTokenizerFactory(Collections.emptyMap()))) - .build(); - return new PerFieldAnalyzerWrapper(defaultAnalyzer, analyzerMap); + return new LimitTokenCountAnalyzer(result, maxFieldLength); } private static Map collectAnalyzers(NodeState defn) { diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java index cb36d2d..fed2cc8 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java @@ -210,4 +210,18 @@ public interface LuceneIndexConstants { * nt:file node under 'tika' node which refers to the config xml file */ String TIKA_CONFIG = "config"; + + /** + * The maximum number of terms that will be indexed for a single field in a + * document. This limits the amount of memory required for indexing, so that + * collections with very large files will not crash the indexing process by + * running out of memory.

+ * Note that this effectively truncates large documents, excluding from the + * index terms that occur further in the document. If you know your source + * documents are large, be sure to set this value high enough to accommodate + * the expected size. If you set it to Integer.MAX_VALUE, then the only limit + * is your memory, but you should anticipate an OutOfMemoryError.

+ * By default, no more than 10,000 terms will be indexed for a field. + */ + String MAX_FIELD_LENGTH = "maxFieldLength"; } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java index 0eaa58d..ec0e173 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java @@ -176,7 +176,7 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { if (addOrUpdate(path, after, before.exists())) { long indexed = context.incIndexedNodes(); if (indexed % 1000 == 0) { - log.debug("Indexed {} nodes...", indexed); + log.debug("{} => Indexed {} nodes...", context.getDefinition(), indexed); } } } @@ -193,7 +193,7 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { "Failed to close the Lucene index", e); } if (context.getIndexedNodes() > 0) { - log.debug("Indexed {} nodes, done.", context.getIndexedNodes()); + log.debug("{} => Indexed {} nodes, done.", context.getDefinition(), context.getIndexedNodes()); } } } diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java index 44e65ed..49dbfba 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java @@ -980,6 +980,25 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { assertQuery("select * from [nt:base] where CONTAINS(*, 'sky ')", Collections.emptyList()); } + @Test + public void maxFieldLengthCheck() throws Exception{ + Tree idx = createFulltextIndex(root.getTree("/"), "test"); + TestUtil.useV2(idx); + + Tree test = root.getTree("/").addChild("test"); + test.setProperty("text", "red brown fox was jumping"); + root.commit(); + + assertQuery("select * from [nt:base] where CONTAINS(*, 'jumping')", asList("/test")); + + idx = root.getTree("/oak:index/test"); + idx.setProperty(LuceneIndexConstants.MAX_FIELD_LENGTH, 2); + idx.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true); + root.commit(); + + assertQuery("select * from [nt:base] where CONTAINS(*, 'jumping')", Collections.emptyList()); + } + private void createFileNode(Tree tree, String name, String content, String mimeType){ Tree jcrContent = tree.addChild(name).addChild(JCR_CONTENT); jcrContent.setProperty(JcrConstants.JCR_DATA, content.getBytes());