diff --git a/oak-search-elastic/pom.xml b/oak-search-elastic/pom.xml index 6d42ae28b0..87dec1ad61 100644 --- a/oak-search-elastic/pom.xml +++ b/oak-search-elastic/pom.xml @@ -33,8 +33,8 @@ Oak Elasticsearch integration subproject - 7.1.1 - 8.0.0 + 7.6.0 + 8.4.0 @@ -44,13 +44,30 @@ maven-bundle-plugin - - + <_exportcontents> + org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexProviderService + + + !com.carrotsearch.randomizedtesting.*, + !com.google.common.geometry.*, + !com.sun.management.*, + !jdk.net.*, + !org.apache.avalon.framework.logger.*, + !org.apache.log.*, + !org.apache.logging.*, + !org.elasticsearch.geometry.*, + !org.joda.convert.*, + !org.locationtech.jts.geom.*, + !org.locationtech.spatial4j.*, + !sun.misc.*, + * + - lucene-core;inline=true - elasticsearch-rest-high-level-client;inline=true + oak-search;inline=true, + elasticsearch;groupId=org.elasticsearch, + *;scope=compile|runtime - !* + true @@ -91,75 +108,90 @@ provided + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + org.elasticsearch.client + elasticsearch-rest-client + ${elasticsearch.version} + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + + + org.elasticsearch + elasticsearch-x-content + ${elasticsearch.version} + + + org.apache.lucene + lucene-core + ${lucene.version} + + org.apache.jackrabbit oak-api ${project.version} + provided org.apache.jackrabbit oak-core-spi ${project.version} + provided org.apache.jackrabbit oak-store-spi ${project.version} + provided org.apache.jackrabbit oak-query-spi ${project.version} + provided org.apache.jackrabbit oak-core ${project.version} - - - org.apache.jackrabbit - oak-store-document - ${project.version} + provided org.apache.jackrabbit oak-search ${project.version} + provided org.slf4j slf4j-api + provided org.jetbrains annotations + provided - - ch.qos.logback - logback-classic - test - junit junit test - - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${elasticsearch.version} - - - org.apache.lucene - lucene-core - ${lucene.version} - org.apache.jackrabbit oak-core @@ -169,132 +201,27 @@ org.apache.jackrabbit - oak-blob-plugins - ${project.version} - tests - test - - - org.apache.jackrabbit - oak-store-spi - ${project.version} - test-jar - test - - - org.apache.jackrabbit - oak-segment-tar - ${project.version} - test - - - org.apache.jackrabbit - oak-segment-tar - ${project.version} - test-jar - test - - - org.apache.jackrabbit - oak-store-document - ${project.version} - test-jar - test - - - org.apache.jackrabbit - oak-jcr - ${project.version} - test - - - org.apache.jackrabbit - oak-jcr + oak-security-spi ${project.version} - test-jar test - - org.apache.jackrabbit - oak-commons - ${project.version} - test-jar - test - - - org.apache.jackrabbit - jackrabbit-jcr-tests - ${jackrabbit.version} - test - - - org.apache.jackrabbit - jackrabbit-core - ${jackrabbit.version} - tests - test - - - org.apache.tika - tika-parsers - ${tika.version} - test - - - commons-logging - commons-logging - - - org.slf4j - slf4j-log4j12 - - - org.apache.sling org.apache.sling.testing.osgi-mock - - - org.hamcrest - hamcrest-all - 1.3 test - org.mockito - mockito-core - test - - - io.dropwizard.metrics - metrics-core - test - - - org.apache.commons - commons-exec + org.hamcrest + hamcrest-core 1.3 test - com.google.code.gson - gson - 2.8.0 - test - - - com.arakelian - docker-junit-rule - test - - - org.apache.jackrabbit - oak-search - tests - ${project.version} + org.testcontainers + elasticsearch + 1.12.5 test - diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/DefaultElasticsearchIndexCoordinateFactory.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/DefaultElasticsearchIndexCoordinateFactory.java deleted file mode 100644 index 5328b53028..0000000000 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/DefaultElasticsearchIndexCoordinateFactory.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index.elasticsearch; - -import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; -import org.apache.jackrabbit.oak.spi.state.NodeState; - -import java.util.Map; - -public class DefaultElasticsearchIndexCoordinateFactory implements ElasticsearchIndexCoordinateFactory { - private final Map config; - private final ElasticsearchConnectionFactory factory; - - DefaultElasticsearchIndexCoordinateFactory(ElasticsearchConnectionFactory factory, Map config) { - this.factory = factory; - this.config = config; - } - - @Override - public ElasticsearchIndexCoordinate getElasticsearchIndexCoordinate(IndexDefinition indexDefinition) { - ElasticsearchCoordinate esCoord = getElasticsearchCoordinate(indexDefinition.getDefinitionNodeState()); - return new ElasticsearchIndexCoordinateImpl(esCoord, indexDefinition); - } - - private ElasticsearchCoordinate getElasticsearchCoordinate(NodeState indexDefinition) { - return ElasticsearchCoordinateImpl.construct(factory, indexDefinition, config); - } -} diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnection.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnection.java new file mode 100644 index 0000000000..03e37f3f2d --- /dev/null +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnection.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.elasticsearch; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +/** + * This class represents an Elasticsearch Connection with the related RestHighLevelClient. + * As per Elasticsearch documentation: the client is thread-safe, there should be one instance per application and it + * must be closed when it is not needed anymore. + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/_changing_the_client_8217_s_initialization_code.html + * + * The getClient() initializes the rest client on the first call. + * Once close() is invoked this instance cannot be used anymore. + */ +public class ElasticsearchConnection implements Closeable { + + protected static final String SCHEME_PROP = "elasticsearch.scheme"; + protected static final String DEFAULT_SCHEME = "http"; + protected static final String HOST_PROP = "elasticsearch.host"; + protected static final String DEFAULT_HOST = "127.0.0.1"; + protected static final String PORT_PROP = "elasticsearch.port"; + protected static final int DEFAULT_PORT = 9200; + + protected static final Supplier defaultConnection = () -> + new ElasticsearchConnection(DEFAULT_SCHEME, DEFAULT_HOST, DEFAULT_PORT); + + private String scheme; + private String host; + private int port; + + private volatile RestHighLevelClient client; + + private AtomicBoolean isClosed = new AtomicBoolean(false); + + protected ElasticsearchConnection(String scheme, String host, Integer port) { + if (scheme == null || host == null || port == null) { + throw new IllegalArgumentException(); + } + this.scheme = scheme; + this.host = host; + this.port = port; + } + + public RestHighLevelClient getClient() { + if (isClosed.get()) { + throw new IllegalStateException("Already closed"); + } + + // double checked locking to get good performance and avoid double initialization + if (client == null) { + synchronized (this) { + if (client == null) { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(host, port, scheme))); + } + } + } + return client; + } + + public String getScheme() { + return scheme; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } + + @Override + public synchronized void close() throws IOException { + if (client != null) { + client.close(); + } + isClosed.set(true); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ElasticsearchConnection that = (ElasticsearchConnection) o; + return port == that.port && + Objects.equals(scheme, that.scheme) && + Objects.equals(host, that.host); + } + + @Override + public int hashCode() { + return Objects.hash(getScheme(), getHost(), getPort()); + } + + @Override + public String toString() { + return getScheme() + "://" + getHost() + ":" + getPort(); + } + +} diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnectionFactory.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnectionFactory.java deleted file mode 100644 index 1894777e5b..0000000000 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnectionFactory.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index.elasticsearch; - -import com.google.common.collect.Maps; -import org.apache.http.HttpHost; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.IOException; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -public class ElasticsearchConnectionFactory implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchConnectionFactory.class); - private final ConcurrentMap clientMap = Maps.newConcurrentMap(); - - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - private final AtomicBoolean isClosed = new AtomicBoolean(); - - public RestHighLevelClient getConnection(ElasticsearchCoordinate esCoord) { - lock.readLock().lock(); - try { - if (isClosed.get()) { - throw new IllegalStateException("Already closed"); - } - - return clientMap.computeIfAbsent(esCoord, elasticsearchCoordinate -> { - LOG.info("Creating client {}", elasticsearchCoordinate); - return new RestHighLevelClient( - RestClient.builder( - new HttpHost(elasticsearchCoordinate.getHost(), - elasticsearchCoordinate.getPort(), - elasticsearchCoordinate.getScheme()) - )); - }); - } finally { - lock.readLock().unlock(); - } - } - - @Override - public void close() { - lock.writeLock().lock(); - try { - isClosed.set(true); - clientMap.values().forEach(restHighLevelClient -> { - try { - restHighLevelClient.close(); - } catch (IOException e) { - LOG.error("Error occurred while closing a connection", e); - } - }); - } finally { - lock.writeLock().unlock(); - } - } -} diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchCoordinate.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchCoordinate.java deleted file mode 100644 index 97f87b5ef8..0000000000 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchCoordinate.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index.elasticsearch; - -import org.elasticsearch.client.RestHighLevelClient; - -public interface ElasticsearchCoordinate { - String SCHEME_PROP = "elasticsearch.scheme"; - String DEFAULT_SCHEME = "http"; - String HOST_PROP = "elasticsearch.host"; - String DEFAULT_HOST = "127.0.0.1"; - String PORT_PROP = "elasticsearch.port"; - int DEFAULT_PORT = 9200; - - RestHighLevelClient getClient(); - String getScheme(); - String getHost(); - int getPort(); -} diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchCoordinateImpl.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchCoordinateImpl.java deleted file mode 100644 index 7d99e9b55e..0000000000 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchCoordinateImpl.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index.elasticsearch; - -import com.google.common.base.Objects; -import org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil; -import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.elasticsearch.client.RestHighLevelClient; - -import java.util.Map; - -public class ElasticsearchCoordinateImpl implements ElasticsearchCoordinate { - private final ElasticsearchConnectionFactory connectionFactory; - private final String scheme; - private final String host; - private final int port; - - ElasticsearchCoordinateImpl(ElasticsearchConnectionFactory connectionFactory, - String scheme, String host, int port) { - this.connectionFactory = connectionFactory; - this.scheme = scheme; - this.host = host; - this.port = port; - } - - static ElasticsearchCoordinate construct(ElasticsearchConnectionFactory connectionFactory, - NodeState indexDefn, Map configMap) { - ElasticsearchCoordinate esCoord; - - // index defn is at highest prio - esCoord = readFrom(connectionFactory, indexDefn); - if (esCoord != null) { - return esCoord; - } - - // command line comes next - esCoord = construct(connectionFactory, - System.getProperty(SCHEME_PROP), System.getProperty(HOST_PROP), Integer.getInteger(PORT_PROP)); - if (esCoord != null) { - return esCoord; - } - - // config map - if (configMap != null) { - Integer port = null; - try { - port = Integer.parseInt(configMap.get(PORT_PROP)); - } catch (NumberFormatException nfe) { - // ignore - } - esCoord = construct(connectionFactory, configMap.get(SCHEME_PROP), configMap.get(HOST_PROP), port); - if (esCoord != null) { - return esCoord; - } - } - - return new ElasticsearchCoordinateImpl(connectionFactory, DEFAULT_SCHEME, DEFAULT_HOST, DEFAULT_PORT); - } - - @Override - public RestHighLevelClient getClient() { - return connectionFactory.getConnection(this); - } - - @Override - public String getScheme() { - return scheme; - } - - @Override - public String getHost() { - return host; - } - - @Override - public int getPort() { - return port; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof ElasticsearchCoordinate)) { - return false; - } - - ElasticsearchCoordinate other = (ElasticsearchCoordinate)o; - return hashCode() == other.hashCode() // just to have a quicker comparison - && getScheme().equals(other.getScheme()) - && getHost().equals(other.getHost()) - && getPort() == other.getPort(); - } - - @Override - public int hashCode() { - return Objects.hashCode(getScheme(), getHost(), getPort()); - } - - @Override - public String toString() { - return getScheme() + "://" + getHost() + ":" + getPort(); - } - - private static ElasticsearchCoordinate readFrom(ElasticsearchConnectionFactory factory, NodeState definition) { - if (definition == null - || !definition.hasProperty(SCHEME_PROP) - || !definition.hasProperty(HOST_PROP) - || !definition.hasProperty(PORT_PROP)) { - return null; - } - - String scheme = ConfigUtil.getOptionalValue(definition, SCHEME_PROP, DEFAULT_SCHEME); - String host = ConfigUtil.getOptionalValue(definition, HOST_PROP, DEFAULT_HOST); - int port = ConfigUtil.getOptionalValue(definition, PORT_PROP, DEFAULT_PORT); - - return new ElasticsearchCoordinateImpl(factory, scheme, host, port); - } - - private static ElasticsearchCoordinate construct(ElasticsearchConnectionFactory factory, - String scheme, String host, Integer port) { - if (scheme == null || host == null || port == null) { - return null; - } - - return new ElasticsearchCoordinateImpl(factory, scheme, host, port); - } -} diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinate.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinate.java deleted file mode 100644 index 7e63aa0d8b..0000000000 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinate.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index.elasticsearch; - -import org.elasticsearch.client.RestHighLevelClient; - -public interface ElasticsearchIndexCoordinate { - RestHighLevelClient getClient(); - String getEsIndexName(); -} diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinateFactory.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinateFactory.java deleted file mode 100644 index 5b9626e21a..0000000000 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinateFactory.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index.elasticsearch; - -import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; - -public interface ElasticsearchIndexCoordinateFactory { - ElasticsearchIndexCoordinate getElasticsearchIndexCoordinate(IndexDefinition indexDefinition); -} diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinateImpl.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDescriptor.java similarity index 52% rename from oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinateImpl.java rename to oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDescriptor.java index 0889365c38..cac4015d62 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinateImpl.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDescriptor.java @@ -16,68 +16,65 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch; -import com.google.common.base.Joiner; -import com.google.common.base.Objects; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; import org.elasticsearch.client.RestHighLevelClient; import org.jetbrains.annotations.NotNull; -import java.util.Collections; -import java.util.List; +import java.util.Objects; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import static org.elasticsearch.common.Strings.INVALID_FILENAME_CHARS; -public class ElasticsearchIndexCoordinateImpl implements ElasticsearchIndexCoordinate { +public class ElasticsearchIndexDescriptor { private static final int MAX_NAME_LENGTH = 255; - private final ElasticsearchCoordinate esCoord; - private final String esIndexName; + private static final String INVALID_CHARS_REGEX = Pattern.quote(INVALID_FILENAME_CHARS + .stream() + .map(Object::toString) + .collect(Collectors.joining(""))); + + private final ElasticsearchConnection connection; + private final String indexName; - ElasticsearchIndexCoordinateImpl(@NotNull ElasticsearchCoordinate esCoord, IndexDefinition indexDefinition) { - this.esCoord = esCoord; - esIndexName = getRemoteIndexName(indexDefinition, indexDefinition.getIndexPath()); + public ElasticsearchIndexDescriptor(@NotNull ElasticsearchConnection connection, IndexDefinition indexDefinition) { + this.connection = connection; + this.indexName = getRemoteIndexName(indexDefinition); } - @Override public RestHighLevelClient getClient() { - return esCoord.getClient(); + return connection.getClient(); } - @Override - public String getEsIndexName() { - return esIndexName; + public String getIndexName() { + return indexName; } @Override public int hashCode() { - return Objects.hashCode(esCoord, esIndexName); + return Objects.hash(connection, indexName); } @Override public boolean equals(Object o) { - if (! (o instanceof ElasticsearchIndexCoordinateImpl)) { - return false; - } - ElasticsearchIndexCoordinateImpl other = (ElasticsearchIndexCoordinateImpl)o; - return hashCode() == other.hashCode() - && esCoord.equals(other.esCoord) - && esIndexName.equals(other.esIndexName); + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ElasticsearchIndexDescriptor that = (ElasticsearchIndexDescriptor) o; + return connection.equals(that.connection) && + indexName.equals(that.indexName); } - private String getRemoteIndexName(IndexDefinition definition, String indexPath) { + private String getRemoteIndexName(IndexDefinition definition) { String suffix = definition.getUniqueId(); if (suffix == null) { suffix = String.valueOf(definition.getReindexCount()); } - return getESSafeIndexName(indexPath + "-" + suffix); + return getESSafeIndexName(definition.getIndexPath() + "-" + suffix); } /** @@ -86,25 +83,17 @@ public class ElasticsearchIndexCoordinateImpl implements ElasticsearchIndexCoord *
  • xy:abc -> xyabc
    • *
  • /oak:index/abc -> abc
    • * - * + *

    * The resulting file name would be truncated to MAX_NAME_LENGTH */ private static String getESSafeIndexName(String indexPath) { - List elements = Lists.newArrayList(PathUtils.elements(indexPath)); - Collections.reverse(elements); - List result = Lists.newArrayListWithCapacity(2); - - //Max 3 nodeNames including oak:index which is the immediate parent for any indexPath - for (String e : Iterables.limit(elements, 3)) { - if ("oak:index".equals(e)) { - continue; - } - - result.add(getESSafeName(e)); - } + String name = StreamSupport + .stream(PathUtils.elements(indexPath).spliterator(), false) + .limit(3) //Max 3 nodeNames including oak:index which is the immediate parent for any indexPath + .filter(p -> !"oak:index".equals(p)) + .map(ElasticsearchIndexDescriptor::getESSafeName) + .collect(Collectors.joining("_")); - Collections.reverse(result); - String name = Joiner.on('_').join(result); if (name.length() > MAX_NAME_LENGTH) { name = name.substring(0, MAX_NAME_LENGTH); } @@ -116,11 +105,7 @@ public class ElasticsearchIndexCoordinateImpl implements ElasticsearchIndexCoord * Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html */ private static String getESSafeName(String suggestedIndexName) { - String invalidCharsRegex = Pattern.quote(INVALID_FILENAME_CHARS - .stream() - .map(Object::toString) - .collect(Collectors.joining(""))); - return suggestedIndexName.replaceAll(invalidCharsRegex, "").toLowerCase(); + return suggestedIndexName.replaceAll(INVALID_CHARS_REGEX, "").toLowerCase(); } } diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderService.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderService.java index fd6a2a969f..7d3789fca7 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderService.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderService.java @@ -16,11 +16,15 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.commons.io.FilenameUtils; -import org.apache.felix.scr.annotations.*; +import org.apache.felix.scr.annotations.Activate; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Deactivate; +import org.apache.felix.scr.annotations.Property; +import org.apache.felix.scr.annotations.Reference; +import org.apache.felix.scr.annotations.ReferenceCardinality; +import org.apache.felix.scr.annotations.ReferencePolicy; +import org.apache.felix.scr.annotations.ReferencePolicyOption; import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean; import org.apache.jackrabbit.oak.cache.CacheStats; import org.apache.jackrabbit.oak.commons.IOUtils; @@ -31,22 +35,24 @@ import org.apache.jackrabbit.oak.plugins.index.elasticsearch.index.Elasticsearch import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchIndexProvider; import org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider; import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache; -import org.apache.jackrabbit.oak.plugins.index.search.TextExtractionStatsMBean; +import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.whiteboard.Registration; import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.jetbrains.annotations.NotNull; import org.osgi.framework.BundleContext; import org.osgi.framework.ServiceRegistration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.ArrayList; import java.util.Dictionary; import java.util.Hashtable; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; -import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.commons.io.FileUtils.ONE_MB; import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.registerMBean; @@ -83,26 +89,23 @@ public class ElasticsearchIndexProviderService { ) private static final String PROP_PRE_EXTRACTED_TEXT_ALWAYS_USE = "alwaysUsePreExtractedCache"; - private static final String PROP_ELASTICSEARCH_SCHEME_DEFAULT = "http"; -// @Property( -// value = PROP_ELASTICSEARCH_SCHEME_DEFAULT, -// label = "Elasticsearch connection scheme" -// ) - private static final String PROP_ELASTICSEARCH_SCHEME = "elasticsearch.scheme"; - - private static final String PROP_ELASTICSEARCH_HOST_DEFAULT = "localhost"; -// @Property( -// value = PROP_ELASTICSEARCH_HOST_DEFAULT, -// label = "Elasticsearch connection host" -// ) - private static final String PROP_ELASTICSEARCH_HOST = "elasticsearch.host"; - - private static final int PROP_ELASTICSEARCH_PORT_DEFAULT = 9200; -// @Property( -// intValue = PROP_ELASTICSEARCH_PORT_DEFAULT, -// label = "Elasticsearch connection port" -// ) - private static final String PROP_ELASTICSEARCH_PORT = "elasticsearch.port"; + @Property( + value = ElasticsearchConnection.DEFAULT_SCHEME, + label = "Elasticsearch connection scheme" + ) + private static final String PROP_ELASTICSEARCH_SCHEME = ElasticsearchConnection.SCHEME_PROP; + + @Property( + value = ElasticsearchConnection.DEFAULT_HOST, + label = "Elasticsearch connection host" + ) + private static final String PROP_ELASTICSEARCH_HOST = ElasticsearchConnection.HOST_PROP; + + @Property( + intValue = ElasticsearchConnection.DEFAULT_PORT, + label = "Elasticsearch connection port" + ) + private static final String PROP_ELASTICSEARCH_PORT = ElasticsearchConnection.PORT_PROP; @Property( label = "Local text extraction cache path", @@ -121,26 +124,25 @@ public class ElasticsearchIndexProviderService { private ExtractedTextCache extractedTextCache; - private ElasticsearchConnectionFactory connectionFactory = null; - - private final List regs = Lists.newArrayList(); - private final List oakRegs = Lists.newArrayList(); + private final List regs = new ArrayList<>(); + private final List oakRegs = new ArrayList<>(); private Whiteboard whiteboard; private File textExtractionDir; + private ElasticsearchConnection elasticsearchConnection; + @Activate private void activate(BundleContext bundleContext, Map config) { whiteboard = new OsgiWhiteboard(bundleContext); - initializeTextExtractionDir(bundleContext, config); - initializeExtractedTextCache(config, statisticsProvider); + //initializeTextExtractionDir(bundleContext, config); + //initializeExtractedTextCache(config, statisticsProvider); - connectionFactory = new ElasticsearchConnectionFactory(); - ElasticsearchIndexCoordinateFactory esIndexCoordFactory = getElasticsearchIndexCoordinateFactory(config); + elasticsearchConnection = getElasticsearchCoordinate(config); - registerIndexProvider(bundleContext, esIndexCoordFactory); - registerIndexEditor(bundleContext, esIndexCoordFactory); + registerIndexProvider(bundleContext); + registerIndexEditor(bundleContext); } @Deactivate @@ -153,33 +155,32 @@ public class ElasticsearchIndexProviderService { reg.unregister(); } - IOUtils.closeQuietly(connectionFactory); - connectionFactory = null; + IOUtils.closeQuietly(elasticsearchConnection); if (extractedTextCache != null) { extractedTextCache.close(); } } - private void registerIndexProvider(BundleContext bundleContext, ElasticsearchIndexCoordinateFactory esIndexCoordFactory) { - ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(esIndexCoordFactory); + private void registerIndexProvider(BundleContext bundleContext) { + ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(elasticsearchConnection); Dictionary props = new Hashtable<>(); props.put("type", ElasticsearchIndexConstants.TYPE_ELASTICSEARCH); - regs.add(bundleContext.registerService(IndexEditorProvider.class.getName(), indexProvider, props)); + regs.add(bundleContext.registerService(QueryIndexProvider.class.getName(), indexProvider, props)); } - private void registerIndexEditor(BundleContext bundleContext, ElasticsearchIndexCoordinateFactory esIndexCoordFactory) { - ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(esIndexCoordFactory, extractedTextCache); + private void registerIndexEditor(BundleContext bundleContext) { + ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(elasticsearchConnection, extractedTextCache); Dictionary props = new Hashtable<>(); props.put("type", ElasticsearchIndexConstants.TYPE_ELASTICSEARCH); regs.add(bundleContext.registerService(IndexEditorProvider.class.getName(), editorProvider, props)); - oakRegs.add(registerMBean(whiteboard, - TextExtractionStatsMBean.class, - editorProvider.getExtractedTextCache().getStatsMBean(), - TextExtractionStatsMBean.TYPE, - "TextExtraction statistics")); +// oakRegs.add(registerMBean(whiteboard, +// TextExtractionStatsMBean.class, +// editorProvider.getExtractedTextCache().getStatsMBean(), +// TextExtractionStatsMBean.TYPE, +// "TextExtraction statistics")); } private void initializeExtractedTextCache(Map config, StatisticsProvider statisticsProvider) { @@ -209,17 +210,19 @@ public class ElasticsearchIndexProviderService { } } - void initializeTextExtractionDir(BundleContext bundleContext, Map config) { + private void initializeTextExtractionDir(BundleContext bundleContext, Map config) { String textExtractionDir = PropertiesUtil.toString(config.get(PROP_LOCAL_TEXT_EXTRACTION_DIR), null); - if (Strings.isNullOrEmpty(textExtractionDir)) { + if (textExtractionDir == null || textExtractionDir.trim().isEmpty()) { String repoHome = bundleContext.getProperty(REPOSITORY_HOME); if (repoHome != null) { textExtractionDir = FilenameUtils.concat(repoHome, "index"); } } - checkNotNull(textExtractionDir, "Text extraction directory cannot be determined as neither " + - "directory path [%s] nor repository home [%s] defined", PROP_LOCAL_TEXT_EXTRACTION_DIR, REPOSITORY_HOME); + if (textExtractionDir == null) { + throw new IllegalStateException(String.format("Text extraction directory cannot be determined as neither " + + "directory path [%s] nor repository home [%s] defined", PROP_LOCAL_TEXT_EXTRACTION_DIR, REPOSITORY_HOME)); + } this.textExtractionDir = new File(textExtractionDir); } @@ -238,16 +241,35 @@ public class ElasticsearchIndexProviderService { } } - private ElasticsearchIndexCoordinateFactory getElasticsearchIndexCoordinateFactory(Map config) { - ElasticsearchIndexCoordinateFactory esIndexCoordFactory; - Map esCfg = Maps.newHashMap(); - esCfg.put(ElasticsearchCoordinate.SCHEME_PROP, - PropertiesUtil.toString(config.get(PROP_ELASTICSEARCH_SCHEME), PROP_ELASTICSEARCH_SCHEME_DEFAULT)); - esCfg.put(ElasticsearchCoordinate.HOST_PROP, - PropertiesUtil.toString(config.get(PROP_ELASTICSEARCH_HOST), PROP_ELASTICSEARCH_HOST_DEFAULT)); - esCfg.put(ElasticsearchCoordinate.PORT_PROP, String.valueOf( - PropertiesUtil.toInteger(config.get(PROP_ELASTICSEARCH_PORT), PROP_ELASTICSEARCH_PORT_DEFAULT))); - esIndexCoordFactory = new DefaultElasticsearchIndexCoordinateFactory(connectionFactory, esCfg); - return esIndexCoordFactory; + private ElasticsearchConnection getElasticsearchCoordinate(Map contextConfig) { + // system properties have priority + ElasticsearchConnection connection = build(System.getProperties().entrySet() + .stream() + .collect(Collectors.toMap( + e -> String.valueOf(e.getKey()), + e -> String.valueOf(e.getValue())) + ) + ); + + if (connection == null) { + connection = build(contextConfig); + } + + return connection != null ? connection : ElasticsearchConnection.defaultConnection.get(); + } + + private ElasticsearchConnection build(@NotNull Map config) { + ElasticsearchConnection coordinate = null; + Object p = config.get(PROP_ELASTICSEARCH_PORT); + if (p != null) { + try { + Integer port = Integer.parseInt(p.toString()); + coordinate = new ElasticsearchConnection((String) config.get(PROP_ELASTICSEARCH_SCHEME), + (String) config.get(PROP_ELASTICSEARCH_HOST), port); + } catch (NumberFormatException nfe) { + LOG.warn("{} value ({}) cannot be parsed to a valid number", PROP_ELASTICSEARCH_PORT, p); + } + } + return coordinate; } } diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java index ea982c8ecb..524d19911b 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java @@ -16,7 +16,6 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.index; -import com.google.common.collect.Maps; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; import org.elasticsearch.common.Strings; @@ -28,11 +27,11 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; -import static com.google.common.collect.Lists.newArrayList; - public class ElasticsearchDocument { private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchDocument.class); @@ -55,11 +54,11 @@ public class ElasticsearchDocument { LOG.warn("Couldn't encode {} as ES id", path); } this.id = id; - this.fulltext = newArrayList(); - this.suggest = newArrayList(); - this.notNullProps = newArrayList(); - this.nullProps = newArrayList(); - this.properties = Maps.newHashMap(); + this.fulltext = new ArrayList<>(); + this.suggest = new ArrayList<>(); + this.notNullProps = new ArrayList<>(); + this.nullProps = new ArrayList<>(); + this.properties = new HashMap<>(); } void addFulltext(String value) { diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorContext.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorContext.java index ea04cab133..b3a8a2bb4a 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorContext.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorContext.java @@ -26,6 +26,8 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.jetbrains.annotations.Nullable; +import java.io.IOException; + public class ElasticsearchIndexEditorContext extends FulltextIndexEditorContext { ElasticsearchIndexEditorContext(NodeState root, NodeBuilder definition, @Nullable IndexDefinition indexDefinition, @@ -53,7 +55,11 @@ public class ElasticsearchIndexEditorContext extends FulltextIndexEditorContext< // Now, that index definition _might_ have been migrated by super call, it would be ok to // get writer and provision index settings and mappings - getWriter().setProvisioningRequired(); + try { + getWriter().provisionIndex(); + } catch (IOException e) { + throw new IllegalStateException("Unable to provision index", e); + } } @Override diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorProvider.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorProvider.java index b73bd05455..4a55b83662 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorProvider.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorProvider.java @@ -21,7 +21,7 @@ import org.apache.jackrabbit.oak.plugins.index.ContextAwareCallback; import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; import org.apache.jackrabbit.oak.plugins.index.IndexingContext; -import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache; import org.apache.jackrabbit.oak.spi.commit.Editor; @@ -30,17 +30,16 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import static com.google.common.base.Preconditions.checkArgument; import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexConstants.TYPE_ELASTICSEARCH; public class ElasticsearchIndexEditorProvider implements IndexEditorProvider { - private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory; + private final ElasticsearchConnection elasticsearchConnection; private final ExtractedTextCache extractedTextCache; - public ElasticsearchIndexEditorProvider(@NotNull ElasticsearchIndexCoordinateFactory esIndexCoordFactory, + public ElasticsearchIndexEditorProvider(@NotNull ElasticsearchConnection elasticsearchConnection, ExtractedTextCache extractedTextCache) { - this.esIndexCoordFactory = esIndexCoordFactory; + this.elasticsearchConnection = elasticsearchConnection; this.extractedTextCache = extractedTextCache != null ? extractedTextCache : new ExtractedTextCache(0, 0); } @@ -49,15 +48,16 @@ public class ElasticsearchIndexEditorProvider implements IndexEditorProvider { @NotNull NodeBuilder definition, @NotNull NodeState root, @NotNull IndexUpdateCallback callback) throws CommitFailedException { if (TYPE_ELASTICSEARCH.equals(type)) { - checkArgument(callback instanceof ContextAwareCallback, "callback instance not of type " + - "ContextAwareCallback [%s]", callback); + if (!(callback instanceof ContextAwareCallback)) { + throw new IllegalStateException("callback instance not of type ContextAwareCallback [" + callback + "]"); + } IndexingContext indexingContext = ((ContextAwareCallback) callback).getIndexingContext(); String indexPath = indexingContext.getIndexPath(); ElasticsearchIndexDefinition indexDefinition = new ElasticsearchIndexDefinition(root, definition.getNodeState(), indexPath); - ElasticsearchIndexWriterFactory writerFactory = new ElasticsearchIndexWriterFactory(esIndexCoordFactory); + ElasticsearchIndexWriterFactory writerFactory = new ElasticsearchIndexWriterFactory(elasticsearchConnection); ElasticsearchIndexEditorContext context = new ElasticsearchIndexEditorContext(root, definition, indexDefinition, diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java index 348c2a516a..78b739264a 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java @@ -16,8 +16,8 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.index; -import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory; -import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinate; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDescriptor; import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriter; @@ -26,7 +26,6 @@ import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.common.Strings; @@ -49,77 +48,58 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; public class ElasticsearchIndexWriter implements FulltextIndexWriter { private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchIndexWriter.class); - private final ElasticsearchIndexCoordinate esIndexCoord; - private final RestHighLevelClient client; - private boolean shouldProvisionIndex; + private final ElasticsearchIndexDescriptor indexDescriptor; private final boolean isAsync; // TODO: use bulk API - https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-docs-bulk-processor.html ElasticsearchIndexWriter(@NotNull IndexDefinition indexDefinition, - ElasticsearchIndexCoordinateFactory esIndexCoordFactory) { - esIndexCoord = esIndexCoordFactory.getElasticsearchIndexCoordinate(indexDefinition); - client = esIndexCoord.getClient(); + @NotNull ElasticsearchConnection elasticsearchConnection) { + indexDescriptor = new ElasticsearchIndexDescriptor(elasticsearchConnection, indexDefinition); // TODO: ES indexing put another bit delay before docs appear in search. // For test without "async" indexing, we can use following hack BUT those where we // would setup async, we'd need to find another way. isAsync = indexDefinition.getDefinitionNodeState().getProperty("async") != null; - - shouldProvisionIndex = false; } @Override public void updateDocument(String path, ElasticsearchDocument doc) throws IOException { - provisionIndex(); - IndexRequest request = new IndexRequest(esIndexCoord.getEsIndexName()) + IndexRequest request = new IndexRequest(indexDescriptor.getIndexName()) .id(pathToId(path)) // immediate refresh would slow indexing response such that next // search would see the effect of this indexed doc. Must only get // enabled in tests (hopefully there are no non-async indexes in real life) .setRefreshPolicy(isAsync ? NONE : IMMEDIATE) .source(doc.build(), XContentType.JSON); - IndexResponse response = client.index(request, RequestOptions.DEFAULT); + IndexResponse response = indexDescriptor.getClient().index(request, RequestOptions.DEFAULT); LOG.trace("update {} - {}. Response: {}", path, doc, response); } @Override public void deleteDocuments(String path) throws IOException { - provisionIndex(); - DeleteRequest request = new DeleteRequest(esIndexCoord.getEsIndexName()) + DeleteRequest request = new DeleteRequest(indexDescriptor.getIndexName()) .id(pathToId(path)) // immediate refresh would slow indexing response such that next // search would see the effect of this indexed doc. Must only get // enabled in tests (hopefully there are no non-async indexes in real life) .setRefreshPolicy(isAsync ? NONE : IMMEDIATE); - DeleteResponse response = client.delete(request, RequestOptions.DEFAULT); + DeleteResponse response = indexDescriptor.getClient().delete(request, RequestOptions.DEFAULT); LOG.trace("delete {}. Response: {}", path, response); } @Override public boolean close(long timestamp) throws IOException { - provisionIndex(); // TODO : track index updates and return accordingly // TODO : if/when we do async push, this is where to wait for those ops to complete return false; } - /** - * This method won't immediately provision index. But, provision would be done before - * any updates are sent to the index - */ - void setProvisioningRequired() { - shouldProvisionIndex = true; - } - - private void provisionIndex() throws IOException { - if (!shouldProvisionIndex) { - return; - } - - try { - CreateIndexRequest request = new CreateIndexRequest(esIndexCoord.getEsIndexName()); + // TODO: we need to check if the index already exists and in that case we have to figure out if there are + // "breaking changes" in the index definition + protected void provisionIndex() throws IOException { + CreateIndexRequest request = new CreateIndexRequest(indexDescriptor.getIndexName()); // provision settings request.settings(Settings.builder() @@ -157,11 +137,8 @@ public class ElasticsearchIndexWriter implements FulltextIndexWriter { - private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory; + private final ElasticsearchConnection elasticsearchConnection; - ElasticsearchIndexWriterFactory(@NotNull ElasticsearchIndexCoordinateFactory esIndexCoordFactory) { - this.esIndexCoordFactory = esIndexCoordFactory; + ElasticsearchIndexWriterFactory(@NotNull ElasticsearchConnection elasticsearchConnection) { + this.elasticsearchConnection = elasticsearchConnection; } @Override public ElasticsearchIndexWriter newInstance(IndexDefinition definition, NodeBuilder definitionBuilder, boolean reindex) { - return new ElasticsearchIndexWriter(definition, esIndexCoordFactory); + return new ElasticsearchIndexWriter(definition, elasticsearchConnection); } } diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java index ed6c8f87f3..4e103a5252 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java @@ -16,12 +16,12 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query; -import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection; import org.apache.jackrabbit.oak.plugins.index.search.IndexNode; import org.apache.jackrabbit.oak.plugins.index.search.SizeEstimator; -import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator; import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex; import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner; +import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator; import org.apache.jackrabbit.oak.spi.query.Cursor; import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.jackrabbit.oak.spi.query.QueryLimits; @@ -45,11 +45,11 @@ public class ElasticsearchIndex extends FulltextIndex { // higher than some threshold below which the query should rather be answered by something else if possible private static final double MIN_COST = 100.1; - private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory; + private final ElasticsearchConnection elasticsearchConnection; private final NodeState root; - ElasticsearchIndex(@NotNull ElasticsearchIndexCoordinateFactory esIndexCoordFactory, @NotNull NodeState root) { - this.esIndexCoordFactory = esIndexCoordFactory; + ElasticsearchIndex(@NotNull ElasticsearchConnection elasticsearchConnection, @NotNull NodeState root) { + this.elasticsearchConnection = elasticsearchConnection; this.root = root; } @@ -85,9 +85,7 @@ public class ElasticsearchIndex extends FulltextIndex { @Override protected IndexNode acquireIndexNode(String indexPath) { - ElasticsearchIndexNode elasticsearchIndexNode = ElasticsearchIndexNode.fromIndexPath(root, indexPath); - elasticsearchIndexNode.setFactory(esIndexCoordFactory); - return elasticsearchIndexNode; + return new ElasticsearchIndexNode(root, indexPath, elasticsearchConnection); } @Override @@ -104,7 +102,7 @@ public class ElasticsearchIndex extends FulltextIndex { final FulltextIndexPlanner.PlanResult pr = getPlanResult(plan); QueryLimits settings = filter.getQueryLimits(); - Iterator itr = new ElasticsearchResultRowIterator(esIndexCoordFactory, filter, pr, plan, + Iterator itr = new ElasticsearchResultRowIterator(filter, pr, plan, acquireIndexNode(plan), FulltextIndex::shouldInclude, getEstimator(plan.getPlanName())); SizeEstimator sizeEstimator = getSizeEstimator(plan); diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexNode.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexNode.java index 1a90adee0c..11d589ccf2 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexNode.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexNode.java @@ -16,8 +16,9 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query; -import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDescriptor; import org.apache.jackrabbit.oak.plugins.index.search.IndexNode; import org.apache.jackrabbit.oak.plugins.index.search.IndexStatistics; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -28,16 +29,13 @@ import org.jetbrains.annotations.Nullable; public class ElasticsearchIndexNode implements IndexNode { private final ElasticsearchIndexDefinition indexDefinition; - private ElasticsearchIndexCoordinateFactory factory; + private final ElasticsearchIndexDescriptor indexDescriptor; - static ElasticsearchIndexNode fromIndexPath(@NotNull NodeState root, @NotNull String indexPath) { - NodeState indexNS = NodeStateUtils.getNode(root, indexPath); - ElasticsearchIndexDefinition indexDefinition = new ElasticsearchIndexDefinition(root, indexNS, indexPath); - return new ElasticsearchIndexNode(indexDefinition); - } - - private ElasticsearchIndexNode(ElasticsearchIndexDefinition indexDefinition) { - this.indexDefinition = indexDefinition; + protected ElasticsearchIndexNode(@NotNull NodeState root, @NotNull String indexPath, + @NotNull ElasticsearchConnection elasticsearchConnection) { + final NodeState indexNS = NodeStateUtils.getNode(root, indexPath); + this.indexDefinition = new ElasticsearchIndexDefinition(root, indexNS, indexPath); + this.indexDescriptor = new ElasticsearchIndexDescriptor(elasticsearchConnection, indexDefinition); } @Override @@ -50,6 +48,10 @@ public class ElasticsearchIndexNode implements IndexNode { return indexDefinition; } + public ElasticsearchIndexDescriptor getIndexDescriptor() { + return indexDescriptor; + } + @Override public int getIndexNodeId() { // TODO: does it matter that we simply return 0 as there's no observation based _refresh_ going on here @@ -59,10 +61,6 @@ public class ElasticsearchIndexNode implements IndexNode { @Override public @Nullable IndexStatistics getIndexStatistics() { - return new ElasticsearchIndexStatistics(factory.getElasticsearchIndexCoordinate(indexDefinition)); - } - - public void setFactory(ElasticsearchIndexCoordinateFactory factory) { - this.factory = factory; + return new ElasticsearchIndexStatistics(indexDescriptor); } } diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexProvider.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexProvider.java index a09924291a..840c67320a 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexProvider.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexProvider.java @@ -16,24 +16,24 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query; -import com.google.common.collect.ImmutableList; -import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection; import org.apache.jackrabbit.oak.spi.query.QueryIndex; import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.jetbrains.annotations.NotNull; +import java.util.Collections; import java.util.List; public class ElasticsearchIndexProvider implements QueryIndexProvider { - private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory; + private final ElasticsearchConnection elasticsearchConnection; - public ElasticsearchIndexProvider(@NotNull ElasticsearchIndexCoordinateFactory esIndexCoordFactory) { - this.esIndexCoordFactory = esIndexCoordFactory; + public ElasticsearchIndexProvider(ElasticsearchConnection elasticsearchConnection) { + this.elasticsearchConnection = elasticsearchConnection; } @Override public @NotNull List getQueryIndexes(NodeState nodeState) { - return ImmutableList.of(new ElasticsearchIndex(esIndexCoordFactory, nodeState)); + return Collections.singletonList(new ElasticsearchIndex(elasticsearchConnection, nodeState)); } } diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexStatistics.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexStatistics.java index c404174660..caf1eef994 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexStatistics.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexStatistics.java @@ -16,31 +16,28 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query; -import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinate; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDescriptor; import org.apache.jackrabbit.oak.plugins.index.search.IndexStatistics; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; public class ElasticsearchIndexStatistics implements IndexStatistics { - private final ElasticsearchIndexCoordinate elasticsearchIndexCoordinate; + private final ElasticsearchIndexDescriptor elasticsearchIndexDescriptor; - ElasticsearchIndexStatistics(ElasticsearchIndexCoordinate elasticsearchIndexCoordinate) { - this.elasticsearchIndexCoordinate = elasticsearchIndexCoordinate; + ElasticsearchIndexStatistics(ElasticsearchIndexDescriptor elasticsearchIndexDescriptor) { + this.elasticsearchIndexDescriptor = elasticsearchIndexDescriptor; } @Override public int numDocs() { CountRequest countRequest = new CountRequest(); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - countRequest.source(searchSourceBuilder); + countRequest.query(QueryBuilders.matchAllQuery()); try { - CountResponse count = elasticsearchIndexCoordinate.getClient().count(countRequest, RequestOptions.DEFAULT); + CountResponse count = elasticsearchIndexDescriptor.getClient().count(countRequest, RequestOptions.DEFAULT); return (int) count.getCount(); } catch (IOException e) { // ignore failure @@ -51,11 +48,9 @@ public class ElasticsearchIndexStatistics implements IndexStatistics { @Override public int getDocCountFor(String key) { CountRequest countRequest = new CountRequest(); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.existsQuery(key)); - countRequest.source(searchSourceBuilder); + countRequest.query(QueryBuilders.existsQuery(key)); try { - CountResponse count = elasticsearchIndexCoordinate.getClient().count(countRequest, RequestOptions.DEFAULT); + CountResponse count = elasticsearchIndexDescriptor.getClient().count(countRequest, RequestOptions.DEFAULT); return (int) count.getCount(); } catch (IOException e) { // ignore failure diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java index 27088a93a1..ddc5a53e98 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java @@ -16,24 +16,25 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query; -import com.google.common.collect.AbstractIterator; -import com.google.common.collect.Iterables; -import com.google.common.collect.Queues; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.commons.PerfLogger; -import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition; -import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator; import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex; import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner.PlanResult; +import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator; import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.jackrabbit.oak.spi.query.QueryConstants; import org.apache.jackrabbit.oak.spi.query.QueryIndex.IndexPlan; -import org.apache.jackrabbit.oak.spi.query.fulltext.*; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextAnd; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextContains; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextOr; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -48,25 +49,41 @@ import javax.jcr.PropertyType; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; +import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiPredicate; +import java.util.stream.StreamSupport; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES; import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE; import static org.apache.jackrabbit.oak.api.Type.STRING; import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot; import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; -import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.*; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newAncestorQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newDepthQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newMixinTypeQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newNodeTypeQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newNotNullPropQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newNullPropQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newPathQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newPrefixPathQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newPrefixQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newPropertyRestrictionQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newWildcardPathQuery; +import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newWildcardQuery; import static org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex.isNodePath; import static org.apache.jackrabbit.oak.spi.query.QueryConstants.JCR_PATH; import static org.apache.jackrabbit.util.ISO8601.parse; -import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; -public class ElasticsearchResultRowIterator extends AbstractIterator { +public class ElasticsearchResultRowIterator implements Iterator { private static final Logger LOG = LoggerFactory .getLogger(ElasticsearchResultRowIterator.class); private static final PerfLogger PERF_LOGGER = @@ -83,15 +100,13 @@ public class ElasticsearchResultRowIterator extends AbstractIterator queue = Queues.newArrayDeque(); + private final Deque queue = new ArrayDeque<>(); // TODO : find if ES can return dup docs - if so how to avoid // private final Set seenPaths = Sets.newHashSet(); private SearchHit lastDoc; private int nextBatchSize = ELASTICSEARCH_QUERY_BATCH_SIZE; private boolean noDocs = false; - private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory; private final Filter filter; private final PlanResult pr; private final IndexPlan plan; @@ -99,14 +114,12 @@ public class ElasticsearchResultRowIterator extends AbstractIterator qs) { - checkNotNull(qs); if (qs.size() == 1) { // we don't need to worry about all-negatives in a bool query as // BoolQueryBuilder.adjustPureNegative is on by default anyway @@ -460,8 +473,6 @@ public class ElasticsearchResultRowIterator extends AbstractIterator qs, IndexPlan plan, PlanResult planResult) { + final BiPredicate, String> any = (iterable, value) -> + StreamSupport.stream(iterable.spliterator(), false).anyMatch(value::equals); + Filter filter = plan.getFilter(); IndexDefinition defn = planResult.indexDefinition; if (!filter.matchesAllTypes()) { @@ -509,7 +523,7 @@ public class ElasticsearchResultRowIterator extends AbstractIterator container.waitForLog("LicenseService")) - .addContainerConfigurer(builder -> builder.env("discovery.type=single-node")) - .build()); - } - - int getPort() { - return getContainer().getPortBinding("9200/tcp").getPort(); - } - - boolean isDockerAvailable() { - return DOCKER_AVAILABLE; - } -} diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderServiceTest.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderServiceTest.java new file mode 100644 index 0000000000..311df6f3d1 --- /dev/null +++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderServiceTest.java @@ -0,0 +1,62 @@ +package org.apache.jackrabbit.oak.plugins.index.elasticsearch; + +import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard; +import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.mount.Mounts; +import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; +import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.apache.sling.testing.mock.osgi.MockOsgi; +import org.apache.sling.testing.mock.osgi.junit.OsgiContext; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.util.Collections; + +import static org.junit.Assert.assertNotNull; + +public class ElasticsearchIndexProviderServiceTest { + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(new File("target")); + + @Rule + public final OsgiContext context = new OsgiContext(); + + private final ElasticsearchIndexProviderService service = new ElasticsearchIndexProviderService(); + + private Whiteboard wb; + + @Before + public void setUp() { + MountInfoProvider mip = Mounts.newBuilder().build(); + context.registerService(MountInfoProvider.class, mip); + context.registerService(NodeStore.class, new MemoryNodeStore()); + context.registerService(StatisticsProvider.class, StatisticsProvider.NOOP); + + wb = new OsgiWhiteboard(context.bundleContext()); + MockOsgi.injectServices(service, context.bundleContext()); + } + + @Test + public void defaultSetup() throws Exception { + MockOsgi.activate(service, context.bundleContext(), + Collections.singletonMap("localTextExtractionDir", folder.newFolder("localTextExtractionDir").getAbsolutePath()) + ); + + assertNotNull(context.getService(QueryIndexProvider.class)); + assertNotNull(context.getService(IndexEditorProvider.class)); + + assertNotNull(WhiteboardUtils.getServices(wb, Runnable.class)); + + MockOsgi.deactivate(service, context.bundleContext()); + } + +} diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchManagementRule.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchManagementRule.java deleted file mode 100644 index ce8aa7e8a6..0000000000 --- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchManagementRule.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index.elasticsearch; - -import com.google.common.collect.Sets; -import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; -import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.client.RequestOptions; -import org.junit.Assume; -import org.junit.rules.ExternalResource; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Set; - -public class ElasticsearchManagementRule extends ExternalResource - implements ElasticsearchIndexCoordinateFactory { - private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchManagementRule.class); - - private final ElasticsearchDockerRule elasticsearch = new ElasticsearchDockerRule(); - - private final ElasticsearchConnectionFactory connectionFactory = new ElasticsearchConnectionFactory(); - - private final Set indices = Sets.newHashSet(); - - private boolean usingDocker; - - @Override - public Statement apply(Statement base, Description description) { - Statement s = super.apply(base, description); - // see if local instance is available... initialize docker rule only if that's not the case - ElasticsearchCoordinate esCoord = ElasticsearchCoordinateImpl.construct(connectionFactory, - null, null); - if (!ElasticsearchTestUtils.isAvailable(esCoord) && elasticsearch.isDockerAvailable()) { - s = elasticsearch.apply(s, description); - usingDocker = true; - } - - return s; - } - - @Override - public ElasticsearchIndexCoordinate getElasticsearchIndexCoordinate(IndexDefinition indexDefinition) { - ElasticsearchCoordinate esCoord = getElasticsearchCoordinate(indexDefinition.getDefinitionNodeState()); - ElasticsearchIndexCoordinate esIdxCoord = new ElasticsearchIndexCoordinateImpl(esCoord, indexDefinition); - indices.add(esIdxCoord); - return esIdxCoord; - } - - @Override - protected void after() { - deletedIndices(); - connectionFactory.close(); - } - - private ElasticsearchCoordinate getElasticsearchCoordinate(NodeState indexDefinition) { - ElasticsearchCoordinate esCoord = ElasticsearchCoordinateImpl.construct(connectionFactory, - indexDefinition, null); - - if (!ElasticsearchTestUtils.isAvailable(esCoord) && usingDocker) { - int port = elasticsearch.getPort(); - esCoord = new ElasticsearchCoordinateImpl(connectionFactory, "http", "localhost", port); - } - - Assume.assumeTrue(ElasticsearchTestUtils.isAvailable(esCoord)); - - return esCoord; - } - - private void deletedIndices() { - indices.forEach(idxCoord -> { - DeleteIndexRequest request = new DeleteIndexRequest(idxCoord.getEsIndexName()); - try { - idxCoord.getClient().indices().delete(request, RequestOptions.DEFAULT); - - LOG.info("Cleaned up index {}", idxCoord.getEsIndexName()); - - } catch (IOException e) { - LOG.warn("Failed to cleanup index {}", idxCoord.getEsIndexName()); - } - }); - } -} \ No newline at end of file diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchPropertyIndexTest.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchPropertyIndexTest.java index fd1b759b75..8dfedd0965 100644 --- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchPropertyIndexTest.java +++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchPropertyIndexTest.java @@ -33,30 +33,36 @@ import org.apache.jackrabbit.oak.query.AbstractQueryTest; import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.elasticsearch.Version; import org.junit.Assert; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; +import org.testcontainers.elasticsearch.ElasticsearchContainer; import java.util.Arrays; -import java.util.Collections; -import java.util.Set; -import static com.google.common.collect.ImmutableSet.of; -import static org.apache.derby.vti.XmlVTI.asList; +import static java.util.Collections.singletonList; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME; import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROPDEF_PROP_NODE_NAME; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; public class ElasticsearchPropertyIndexTest extends AbstractQueryTest { - @Rule - public ElasticsearchManagementRule esMgmt = new ElasticsearchManagementRule(); + + @ClassRule + public static final ElasticsearchContainer ELASTIC = + new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:" + Version.CURRENT); @Override protected ContentRepository createRepository() { - ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(esMgmt, + ElasticsearchConnection coordinate = new ElasticsearchConnection( + ElasticsearchConnection.DEFAULT_SCHEME, + ELASTIC.getContainerIpAddress(), + ELASTIC.getMappedPort(ElasticsearchConnection.DEFAULT_PORT) + ); + ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(coordinate, new ExtractedTextCache(10 * FileUtils.ONE_MB, 100)); - ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(esMgmt); + ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(coordinate); // remove all indexes to avoid cost competition (essentially a TODO for fixing cost ES cost estimation) NodeBuilder builder = InitialContentHelper.INITIAL_CONTENT.builder(); @@ -81,8 +87,8 @@ public class ElasticsearchPropertyIndexTest extends AbstractQueryTest { @Test public void indexSelection() throws Exception { - setIndex("test1", createIndex(of("propa", "propb"))); - setIndex("test2", createIndex(of("propc"))); + setIndex("test1", createIndex("propa", "propb")); + setIndex("test2", createIndex("propc")); Tree test = root.getTree("/").addChild("test"); test.addChild("a").setProperty("propa", "foo"); @@ -96,16 +102,16 @@ public class ElasticsearchPropertyIndexTest extends AbstractQueryTest { assertThat(explain(propaQuery), containsString("elasticsearch:test1")); assertThat(explain("select [jcr:path] from [nt:base] where [propc] = 'foo'"), containsString("elasticsearch:test2")); - assertQuery(propaQuery, asList("/test/a", "/test/b")); - assertQuery("select [jcr:path] from [nt:base] where [propa] = 'foo2'", asList("/test/c")); - assertQuery("select [jcr:path] from [nt:base] where [propc] = 'foo'", asList("/test/d")); + assertQuery(propaQuery, Arrays.asList("/test/a", "/test/b")); + assertQuery("select [jcr:path] from [nt:base] where [propa] = 'foo2'", singletonList("/test/c")); + assertQuery("select [jcr:path] from [nt:base] where [propc] = 'foo'", singletonList("/test/d")); } //OAK-3825 @Test public void nodeNameViaPropDefinition() throws Exception { //make index - IndexDefinitionBuilder builder = createIndex(Collections.EMPTY_SET); + IndexDefinitionBuilder builder = createIndex(); builder.includedPaths("/test") .evaluatePathRestrictions() .indexRule("nt:base") @@ -126,19 +132,19 @@ public class ElasticsearchPropertyIndexTest extends AbstractQueryTest { String explanation = explain(propabQuery); Assert.assertThat(explanation, containsString("elasticsearch:test1(/oak:index/test1) ")); Assert.assertThat(explanation, containsString("{\"term\":{\":nodeName\":{\"value\":\"foo\",")); - assertQuery(propabQuery, Arrays.asList("/test/foo")); - assertQuery(queryPrefix + "LOCALNAME() = 'bar'", Arrays.asList("/test/sc/bar")); - assertQuery(queryPrefix + "LOCALNAME() LIKE 'foo'", Arrays.asList("/test/foo")); - assertQuery(queryPrefix + "LOCALNAME() LIKE 'camel%'", Arrays.asList("/test/camelCase")); - - assertQuery(queryPrefix + "NAME() = 'bar'", Arrays.asList("/test/sc/bar")); - assertQuery(queryPrefix + "NAME() LIKE 'foo'", Arrays.asList("/test/foo")); - assertQuery(queryPrefix + "NAME() LIKE 'camel%'", Arrays.asList("/test/camelCase")); + assertQuery(propabQuery, singletonList("/test/foo")); + assertQuery(queryPrefix + "LOCALNAME() = 'bar'", singletonList("/test/sc/bar")); + assertQuery(queryPrefix + "LOCALNAME() LIKE 'foo'", singletonList("/test/foo")); + assertQuery(queryPrefix + "LOCALNAME() LIKE 'camel%'", singletonList("/test/camelCase")); + + assertQuery(queryPrefix + "NAME() = 'bar'", singletonList("/test/sc/bar")); + assertQuery(queryPrefix + "NAME() LIKE 'foo'", singletonList("/test/foo")); + assertQuery(queryPrefix + "NAME() LIKE 'camel%'", singletonList("/test/camelCase")); } @Test public void emptyIndex() throws Exception { - setIndex("test1", createIndex(of("propa", "propb"))); + setIndex("test1", createIndex("propa", "propb")); root.commit(); Tree test = root.getTree("/").addChild("test"); @@ -151,7 +157,7 @@ public class ElasticsearchPropertyIndexTest extends AbstractQueryTest { @Test public void propertyExistenceQuery() throws Exception { - setIndex("test1", createIndex(of("propa", "propb"))); + setIndex("test1", createIndex("propa", "propb")); Tree test = root.getTree("/").addChild("test"); test.addChild("a").setProperty("propa", "a"); @@ -162,11 +168,12 @@ public class ElasticsearchPropertyIndexTest extends AbstractQueryTest { assertQuery("select [jcr:path] from [nt:base] where propa is not null", Arrays.asList("/test/a", "/test/b")); } - private static IndexDefinitionBuilder createIndex(Set propNames) { + private static IndexDefinitionBuilder createIndex(String... propNames) { IndexDefinitionBuilder builder = new ElasticsearchIndexDefinitionBuilder().noAsync(); IndexDefinitionBuilder.IndexRule indexRule = builder.indexRule("nt:base"); - propNames.forEach(propName -> indexRule.property(propName).propertyIndex()); - + for (String propName : propNames) { + indexRule.property(propName).propertyIndex(); + } return builder; } diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchTestUtils.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchTestUtils.java deleted file mode 100644 index 967c90ba95..0000000000 --- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchTestUtils.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index.elasticsearch; - -import org.jetbrains.annotations.NotNull; - -import java.net.HttpURLConnection; -import java.net.URL; - -class ElasticsearchTestUtils { - private static String createHealthURL(@NotNull final ElasticsearchCoordinate esCoord) { - return esCoord.getScheme() + "://" + esCoord.getHost() + ":" + esCoord.getPort() + "/_cat/health"; - } - - static boolean isAvailable(final ElasticsearchCoordinate esCoord) { - if (esCoord == null) { - return false; - } - - try { - URL url = new URL(createHealthURL(esCoord)); - - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod("GET"); - - int responseCode = con.getResponseCode(); - - return responseCode == 200; - } catch (Throwable t) { - return false; - } - } -}