diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java index 6eb336a..c4a8a97 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugins.index.lucene; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -45,6 +46,7 @@ import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.namepath.NamePathMapper; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.ConfigUtil; import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain; import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; import org.apache.jackrabbit.oak.plugins.nodetype.ReadOnlyNodeTypeManager; @@ -86,6 +88,7 @@ import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstant import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ORDERED_PROP_NAMES; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_NAME; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_NODE; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TIKA_CONFIG; import static org.apache.jackrabbit.oak.plugins.index.lucene.PropertyDefinition.DEFAULT_BOOST; import static org.apache.jackrabbit.oak.plugins.index.lucene.util.ConfigUtil.getOptionalValue; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; @@ -175,6 +178,8 @@ class IndexDefinition implements Aggregate.AggregateMapper{ private final Map analyzers; + private final boolean hasCustomTikaConfig; + public IndexDefinition(NodeState root, NodeState defn) { this(root, defn, null); } @@ -230,6 +235,7 @@ class IndexDefinition implements Aggregate.AggregateMapper{ this.indexesAllTypes = areAllTypesIndexed(); this.analyzers = collectAnalyzers(defn); this.analyzer = createAnalyzer(); + this.hasCustomTikaConfig = defn.getChildNode(TIKA_CONFIG).exists(); } public boolean isFullTextEnabled() { @@ -310,6 +316,14 @@ class IndexDefinition implements Aggregate.AggregateMapper{ return analyzer; } + public boolean hasCustomTikaConfig(){ + return hasCustomTikaConfig; + } + + public InputStream getTikaConfig(){ + return ConfigUtil.getBlob(definition.getChildNode(TIKA_CONFIG), TIKA_CONFIG).getNewStream(); + } + @Override public String toString() { return "IndexDefinition : " + indexName; diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java index 61a3a14..5d4ad37 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java @@ -200,4 +200,6 @@ public interface LuceneIndexConstants { String ANL_CLASS = "class"; String ANL_NAME = "name"; String ANL_LUCENE_MATCH_VERSION = AbstractAnalysisFactory.LUCENE_MATCH_VERSION_PARAM; + + String TIKA_CONFIG = "tikaConfig"; } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java index 014fb72..bc0d6c1 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java @@ -23,8 +23,10 @@ import static org.apache.lucene.store.NoLockFactory.getNoLockFactory; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.Calendar; +import org.apache.commons.io.IOUtils; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; @@ -36,10 +38,13 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.tika.config.TikaConfig; +import org.apache.tika.exception.TikaException; import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.Parser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.xml.sax.SAXException; public class LuceneIndexEditorContext { @@ -89,7 +94,7 @@ public class LuceneIndexEditorContext { private final IndexWriterConfig config; - private static final Parser parser = new AutoDetectParser(); + private static final Parser defaultParser = new AutoDetectParser(); private final IndexDefinition definition; @@ -103,13 +108,15 @@ public class LuceneIndexEditorContext { private boolean reindex; + private final Parser parser; + LuceneIndexEditorContext(NodeState root, NodeBuilder definition, IndexUpdateCallback updateCallback) { this.definitionBuilder = definition; this.definition = new IndexDefinition(root, definition); this.config = getIndexWriterConfig(this.definition); this.indexedNodes = 0; this.updateCallback = updateCallback; - + this.parser = initializeTikaParser(this.definition); if (this.definition.isOfOldFormat()){ IndexDefinition.updateDefinition(definition); } @@ -172,4 +179,23 @@ public class LuceneIndexEditorContext { public IndexDefinition getDefinition() { return definition; } + + private static Parser initializeTikaParser(IndexDefinition definition) { + if (definition.hasCustomTikaConfig()){ + InputStream is = definition.getTikaConfig(); + try { + TikaConfig config = new TikaConfig(is); + return new AutoDetectParser(config); + } catch (IOException e){ + throw new RuntimeException("Error loading TikaConfig for "+ definition, e); + } catch (SAXException e) { + throw new RuntimeException("Error loading TikaConfig for "+ definition, e); + } catch (TikaException e) { + throw new RuntimeException("Error loading TikaConfig for "+ definition, e); + } finally { + IOUtils.closeQuietly(is); + } + } + return defaultParser; + } } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/NodeStateAnalyzerFactory.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/NodeStateAnalyzerFactory.java index c565a5e..9725d3f 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/NodeStateAnalyzerFactory.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/NodeStateAnalyzerFactory.java @@ -32,11 +32,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; -import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.ConfigUtil; import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain; import org.apache.jackrabbit.oak.plugins.tree.TreeFactory; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -232,19 +232,9 @@ final class NodeStateAnalyzerFactory{ return version; } - /** - * Assumes that given state is of type nt:file and then reads - * the jcr:content/@jcr:data property to get the binary content - */ - private static Blob getBlob(NodeState state, String resourceName){ - NodeState contentNode = state.getChildNode(JcrConstants.JCR_CONTENT); - checkArgument(contentNode.exists(), "Was expecting to find jcr:content node to read resource %s", resourceName); - return contentNode.getProperty(JcrConstants.JCR_DATA).getValue(Type.BINARY); - } - private static CharArraySet loadStopwordSet(NodeState file, String name, Version matchVersion) throws IOException { - Blob blob = getBlob(file, name); + Blob blob = ConfigUtil.getBlob(file, name); Reader stopwords = new InputStreamReader(blob.getNewStream(), IOUtils.CHARSET_UTF_8); try { return WordlistLoader.getWordSet(stopwords, matchVersion); @@ -265,7 +255,7 @@ final class NodeStateAnalyzerFactory{ @Override public InputStream openResource(String resource) throws IOException { if (state.hasChildNode(resource)){ - return getBlob(state.getChildNode(resource), resource).getNewStream(); + return ConfigUtil.getBlob(state.getChildNode(resource), resource).getNewStream(); } return delegate.openResource(resource); } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java index 5c229ae..476f7e1 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ConfigUtil.java @@ -21,10 +21,13 @@ package org.apache.jackrabbit.oak.plugins.index.lucene.util; import com.google.common.primitives.Ints; import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.spi.state.NodeState; +import static com.google.common.base.Preconditions.checkArgument; + public class ConfigUtil { public static boolean getOptionalValue(NodeState definition, String propName, boolean defaultVal){ @@ -56,4 +59,14 @@ public class ConfigUtil { PropertyState ps = nodeState.getProperty(JcrConstants.JCR_PRIMARYTYPE); return (ps == null) ? JcrConstants.NT_BASE : ps.getValue(Type.NAME); } + + /** + * Assumes that given state is of type nt:file and then reads + * the jcr:content/@jcr:data property to get the binary content + */ + public static Blob getBlob(NodeState state, String resourceName){ + NodeState contentNode = state.getChildNode(JcrConstants.JCR_CONTENT); + checkArgument(contentNode.exists(), "Was expecting to find jcr:content node to read resource %s", resourceName); + return contentNode.getProperty(JcrConstants.JCR_DATA).getValue(Type.BINARY); + } } diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java index bf49a72..1355a10 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java @@ -485,6 +485,20 @@ public class IndexDefinitionTest { assertEquals(TokenizerChain.class.getName(), defn.getAnalyzer().getClass().getName()); } + @Test + public void customTikaConfig() throws Exception{ + NodeBuilder defnb = newLuceneIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME), + "lucene", of(TYPENAME_STRING)); + IndexDefinition defn = new IndexDefinition(root, defnb.getNodeState()); + assertFalse(defn.hasCustomTikaConfig()); + + defnb.child(LuceneIndexConstants.TIKA_CONFIG) + .child(JcrConstants.JCR_CONTENT) + .setProperty(JcrConstants.JCR_DATA, "hello".getBytes()); + defn = new IndexDefinition(root, defnb.getNodeState()); + assertTrue(defn.hasCustomTikaConfig()); + } + private static IndexingRule getRule(IndexDefinition defn, String typeName){ return defn.getApplicableIndexingRule(newTree(newNode(typeName))); } diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java index 160524e..b767241 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java @@ -56,6 +56,8 @@ import org.junit.Test; import static com.google.common.collect.ImmutableSet.of; import static java.util.Arrays.asList; +import static org.apache.jackrabbit.JcrConstants.JCR_CONTENT; +import static org.apache.jackrabbit.JcrConstants.JCR_DATA; import static org.apache.jackrabbit.oak.api.QueryEngine.NO_MAPPINGS; import static org.apache.jackrabbit.oak.api.Type.STRINGS; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME; @@ -120,7 +122,8 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { INDEX_DEFINITIONS_NODE_TYPE, Type.NAME); def.setProperty(TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE); def.setProperty(REINDEX_PROPERTY_NAME, true); - def.setProperty(createProperty(LuceneIndexConstants.INCLUDE_PROPERTY_TYPES, of(PropertyType.TYPENAME_STRING), STRINGS)); + def.setProperty(createProperty(LuceneIndexConstants.INCLUDE_PROPERTY_TYPES, + of(PropertyType.TYPENAME_STRING, PropertyType.TYPENAME_BINARY), STRINGS)); return index.getChild(INDEX_DEFINITIONS_NAME).getChild(name); } @@ -425,7 +428,7 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { root.commit(); assertQuery("select [jcr:path] from [nt:base] where propa like 'hum%'", - asList("/test/a", "/test/c")); + asList("/test/a", "/test/c")); assertQuery("select [jcr:path] from [nt:base] where propa like '%ty'", asList("/test/a", "/test/b")); assertQuery("select [jcr:path] from [nt:base] where propa like '%ump%'", @@ -610,8 +613,8 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC)); assertOrderedQuery( - "select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", - getSortedPaths(tuples, OrderDirection.DESC)); + "select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", + getSortedPaths(tuples, OrderDirection.DESC)); } @Test @@ -648,8 +651,8 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC)); assertOrderedQuery( - "select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", - getSortedPaths(tuples, OrderDirection.DESC)); + "select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", + getSortedPaths(tuples, OrderDirection.DESC)); } @Test @@ -687,10 +690,10 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { root.commit(); assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", - getSortedPaths(tuples, OrderDirection.ASC)); + getSortedPaths(tuples, OrderDirection.ASC)); assertOrderedQuery( - "select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", - getSortedPaths(tuples, OrderDirection.DESC)); + "select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", + getSortedPaths(tuples, OrderDirection.DESC)); } @Test @@ -725,7 +728,7 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { assertOrderedQuery( "select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", Lists .newArrayList(Iterables.concat(getSortedPaths(tuples, OrderDirection.DESC), - Lists.newArrayList("/test/n0")))); + Lists.newArrayList("/test/n0")))); } @Test @@ -863,9 +866,9 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { root.commit(); assertQuery( - "select * from [nt:base] where CONTAINS(*, 'fox') and CONTAINS([propb], '\"winter is here\" OR \"summer " - + "is here\"')", - asList("/test/a", "/test/b")); + "select * from [nt:base] where CONTAINS(*, 'fox') and CONTAINS([propb], '\"winter is here\" OR \"summer " + + "is here\"')", + asList("/test/a", "/test/b")); } // OAK-2434 @@ -938,6 +941,49 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { assertOrderedQuery(queryString, asList("/test/b", "/test/c", "/test/a"), XPATH, true); } + @Test + public void customTikaConfig() throws Exception{ + Tree idx = createFulltextIndex(root.getTree("/"), "test"); + TestUtil.useV2(idx); + + Tree test = root.getTree("/").addChild("test"); + createFileNode(test, "text", "fox is jumping", "text/plain"); + createFileNode(test, "xml", "sky is blue", "application/xml"); + root.commit(); + + assertQuery("select * from [nt:base] where CONTAINS(*, 'fox ')", asList("/test/text/jcr:content")); + assertQuery("select * from [nt:base] where CONTAINS(*, 'sky ')", asList("/test/xml/jcr:content")); + + //Now disable extraction for application/xml and see that query + //does not return any result for that + idx = root.getTree("/oak:index/test"); + String tikaConfig = "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " application/xml\n" + + " \n" + + " \n" + + ""; + + idx.addChild(LuceneIndexConstants.TIKA_CONFIG).addChild(JCR_CONTENT).setProperty(JCR_DATA, tikaConfig.getBytes()); + idx.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true); + root.commit(); + + assertQuery("select * from [nt:base] where CONTAINS(*, 'fox ')", asList("/test/text/jcr:content")); + assertQuery("select * from [nt:base] where CONTAINS(*, 'sky ')", Collections.emptyList()); + } + + private void createFileNode(Tree tree, String name, String content, String mimeType){ + Tree jcrContent = tree.addChild(name).addChild(JCR_CONTENT); + jcrContent.setProperty(JcrConstants.JCR_DATA, content.getBytes()); + jcrContent.setProperty(JcrConstants.JCR_MIMETYPE, mimeType); + } + private Tree usc(Tree parent, String childName){ Tree child = parent.addChild(childName); child.setProperty(JcrConstants.JCR_PRIMARYTYPE, "oak:Unstructured", Type.NAME);