diff --git oak-search-elastic/pom.xml oak-search-elastic/pom.xml index 2a8a83cd02..bda5120009 100644 --- oak-search-elastic/pom.xml +++ oak-search-elastic/pom.xml @@ -239,7 +239,7 @@ org.hamcrest hamcrest-core - 1.3 + 2.2 test diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDefinition.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDefinition.java index 3324f5d6c5..8592cd235b 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDefinition.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDefinition.java @@ -18,11 +18,17 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch; +import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition; import org.apache.jackrabbit.oak.spi.state.NodeState; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -56,6 +62,16 @@ public class ElasticsearchIndexDefinition extends IndexDefinition { .map(Object::toString) .collect(Collectors.joining(""))); + private static final Function isAnalyzable; + + static { + int[] NOT_ANALYZED_TYPES = new int[] { + Type.BINARY.tag(), Type.LONG.tag(), Type.DOUBLE.tag(), Type.DECIMAL.tag(), Type.DATE.tag(), Type.BOOLEAN.tag() + }; + Arrays.sort(NOT_ANALYZED_TYPES); // need for binary search + isAnalyzable = type -> Arrays.binarySearch(NOT_ANALYZED_TYPES, type) < 0; + } + private final String remoteIndexName; public final int bulkActions; @@ -66,6 +82,8 @@ public class ElasticsearchIndexDefinition extends IndexDefinition { private final String indexPrefix; private final String remoteAlias; + private final Map> propertiesByName; + public ElasticsearchIndexDefinition(NodeState root, NodeState defn, String indexPath, String indexPrefix) { super(root, getIndexDefinitionState(defn), determineIndexFormatVersion(defn), determineUniqueId(defn), indexPath); boolean isReindex = defn.getBoolean(IndexConstants.REINDEX_PROPERTY_NAME); @@ -78,6 +96,12 @@ public class ElasticsearchIndexDefinition extends IndexDefinition { this.bulkFlushIntervalMs = getOptionalValue(defn, BULK_FLUSH_INTERVAL_MS, BULK_FLUSH_INTERVAL_MS_DEFAULT); this.bulkRetries = getOptionalValue(defn, BULK_RETRIES, BULK_RETRIES_DEFAULT); this.bulkRetriesBackoff = getOptionalValue(defn, BULK_RETRIES_BACKOFF, BULK_RETRIES_BACKOFF_DEFAULT); + + this.propertiesByName = getDefinedRules() + .stream() + .flatMap(rule -> StreamSupport.stream(rule.getProperties().spliterator(), false)) + .filter(pd -> pd.index) // keep only properties that can be indexed + .collect(Collectors.groupingBy(pd -> pd.name)); } /** @@ -99,6 +123,35 @@ public class ElasticsearchIndexDefinition extends IndexDefinition { return remoteIndexName; } + public Map> getPropertiesByName() { + return propertiesByName; + } + + /** + * Returns the keyword field name mapped in Elasticsearch for the specified property name. + * @param propertyName the property name in the index rules + * @return the field name identifier in Elasticsearch + * @throws IllegalArgumentException if the specified name is not part of this {@code ElasticsearchIndexDefinition} + */ + public String getElasticKeyword(String propertyName) { + List propertyDefinitions = propertiesByName.get(propertyName); + if (propertyDefinitions == null) { + throw new IllegalArgumentException(propertyName + " is not part of this ElasticsearchIndexDefinition"); + } + + String field = propertyName; + // it's ok to look at the first property since we are sure they all have the same type + int type = propertyDefinitions.get(0).getType(); + if (isAnalyzable.apply(type) && isAnalyzed(propertyDefinitions)) { + field += ".keyword"; + } + return field; + } + + public boolean isAnalyzed(List propertyDefinitions) { + return propertyDefinitions.stream().anyMatch(pd -> pd.analyzed || pd.fulltextEnabled()); + } + private String setupAlias() { // TODO: implement advanced remote index name strategy that takes into account multiple tenants and re-index process return getESSafeIndexName(indexPrefix + "." + getIndexPath()); diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/ElasticFacetHelper.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/ElasticFacetHelper.java index 6694f0e815..2b9c9ab63d 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/ElasticFacetHelper.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/ElasticFacetHelper.java @@ -20,17 +20,17 @@ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.facets; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchIndexNode; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchSearcher; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration; import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.jackrabbit.oak.spi.query.QueryIndex; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchHit; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; public class ElasticFacetHelper { @@ -58,32 +58,28 @@ public class ElasticFacetHelper { return elasticsearchFacets; } - public static List getAccessibleDocIds(SearchHit[] searchHits, Filter filter) throws UnsupportedEncodingException { + public static List getAccessibleDocIds(SearchHit[] searchHits, Filter filter) { List accessibleDocs = new LinkedList<>(); for (SearchHit searchHit : searchHits) { - String id = searchHit.getId(); - String path = idToPath(id); + final Map sourceMap = searchHit.getSourceAsMap(); + String path = (String) sourceMap.get(FieldNames.PATH); if (filter.isAccessible(path)) { - accessibleDocs.add(id); + accessibleDocs.add(path); } } return accessibleDocs; } - public static int getAccessibleDocCount(Iterator searchHitIterator, Filter filter) throws UnsupportedEncodingException { + public static int getAccessibleDocCount(Iterator searchHitIterator, Filter filter) { int count = 0; while (searchHitIterator.hasNext()) { SearchHit searchHit = searchHitIterator.next(); - String id = searchHit.getId(); - String path = idToPath(id); + final Map sourceMap = searchHit.getSourceAsMap(); + String path = (String) sourceMap.get(FieldNames.PATH); if (filter.isAccessible(path)) { count++; } } return count; } - - public static String idToPath(String id) throws UnsupportedEncodingException { - return URLDecoder.decode(id, "UTF-8"); - } } diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/ElasticsearchFacets.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/ElasticsearchFacets.java index 841d70cc9d..ab3c37f9ef 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/ElasticsearchFacets.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/ElasticsearchFacets.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.facets; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchSearcher; import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex; import org.apache.jackrabbit.oak.spi.query.QueryIndex; @@ -49,7 +50,8 @@ public interface ElasticsearchFacets { * @return A map with facetName as key and List of facets in descending order of facetCount. * @throws IOException */ - Map> getElasticSearchFacets(int numberOfFacets) throws IOException; + Map> getElasticSearchFacets(ElasticsearchIndexDefinition indexDefinition, + int numberOfFacets) throws IOException; /** * We can retrieve Aggregation in a single call to elastic search while querying. Which can then be passed diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/InsecureElasticSearchFacets.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/InsecureElasticSearchFacets.java index fdbceb0fbe..f400404b9d 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/InsecureElasticSearchFacets.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/InsecureElasticSearchFacets.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.facets; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchSearcher; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchSearcherModel; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.ElasticsearchAggregationBuilderUtil; @@ -51,12 +52,13 @@ public class InsecureElasticSearchFacets implements ElasticsearchFacets { } @Override - public Map> getElasticSearchFacets(int numberOfFacets) throws IOException { + public Map> getElasticSearchFacets(ElasticsearchIndexDefinition indexDefinition, + int numberOfFacets) throws IOException { if (elasticsearchAggregationData != null && numberOfFacets <= elasticsearchAggregationData.getNumberOfFacets()) { return changeToFacetList(elasticsearchAggregationData.getAggregations().getAsMap(), numberOfFacets); } LOG.warn("Facet data is being retrieved by again calling Elasticsearch"); - List aggregationBuilders = ElasticsearchAggregationBuilderUtil.getAggregators(plan, numberOfFacets); + List aggregationBuilders = ElasticsearchAggregationBuilderUtil.getAggregators(plan, indexDefinition, numberOfFacets); ElasticsearchSearcherModel elasticsearchSearcherModel = new ElasticsearchSearcherModel.ElasticsearchSearcherModelBuilder() .withQuery(query) .withAggregation(aggregationBuilders) diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/SecureElasticSearchFacets.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/SecureElasticSearchFacets.java index b185749ef7..0e1cc1bb34 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/SecureElasticSearchFacets.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/SecureElasticSearchFacets.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.facets; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchSearcher; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchSearcherModel; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.ElasticsearchAggregationBuilderUtil; @@ -50,7 +51,8 @@ public class SecureElasticSearchFacets extends InsecureElasticSearchFacets { for docs. */ @Override - public Map> getElasticSearchFacets(int numberOfFacets) throws IOException { + public Map> getElasticSearchFacets(ElasticsearchIndexDefinition indexDefinition, + int numberOfFacets) throws IOException { Map> secureFacetCount = new HashMap<>(); Filter filter = getPlan().getFilter(); boolean doFetch = true; @@ -70,7 +72,8 @@ public class SecureElasticSearchFacets extends InsecureElasticSearchFacets { List accessibleDocs = ElasticFacetHelper.getAccessibleDocIds(searchHits, filter); if (accessibleDocs.isEmpty()) continue; QueryBuilder queryWithAccessibleDocIds = QueryBuilders.termsQuery("_id", accessibleDocs); - Map accessibleDocsAggregation = getAggregationForDocIds(queryWithAccessibleDocIds, accessibleDocs.size()); + Map accessibleDocsAggregation = getAggregationForDocIds(queryWithAccessibleDocIds, + accessibleDocs.size(), indexDefinition); collateAggregations(secureFacetCount, accessibleDocsAggregation); } @@ -115,15 +118,15 @@ public class SecureElasticSearchFacets extends InsecureElasticSearchFacets { } } - private Map getAggregationForDocIds(QueryBuilder queryWithAccessibleDocIds, int facetCount) throws IOException { - List aggregationBuilders = ElasticsearchAggregationBuilderUtil.getAggregators(getPlan(), facetCount); + private Map getAggregationForDocIds(QueryBuilder queryWithAccessibleDocIds, int facetCount, + ElasticsearchIndexDefinition indexDefinition) throws IOException { + List aggregationBuilders = ElasticsearchAggregationBuilderUtil.getAggregators(getPlan(), indexDefinition, facetCount); ElasticsearchSearcherModel idBasedelasticsearchSearcherModelWithAggregation = new ElasticsearchSearcherModel.ElasticsearchSearcherModelBuilder() .withQuery(queryWithAccessibleDocIds) .withAggregation(aggregationBuilders) .build(); SearchResponse facetDocs = getSearcher().search(idBasedelasticsearchSearcherModelWithAggregation); - Map aggregationMap = facetDocs.getAggregations().asMap(); - return aggregationMap; + return facetDocs.getAggregations().asMap(); } } diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/StatisticalElasticSearchFacets.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/StatisticalElasticSearchFacets.java index d2b0f6fc96..b2c0627693 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/StatisticalElasticSearchFacets.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/facets/StatisticalElasticSearchFacets.java @@ -17,6 +17,7 @@ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.facets; import com.google.common.collect.AbstractIterator; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchSearcher; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchSearcherModel; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.ElasticsearchConstants; @@ -52,7 +53,9 @@ public class StatisticalElasticSearchFacets extends InsecureElasticSearchFacets this.secureFacetConfiguration = secureFacetConfiguration; } - public Map> getElasticSearchFacets(int numberOfFacets) throws IOException { + @Override + public Map> getElasticSearchFacets(ElasticsearchIndexDefinition indexDefinition, + int numberOfFacets) throws IOException { Map> result = new HashMap<>(); Map> topChildren; Filter filter = getPlan().getFilter(); @@ -61,7 +64,7 @@ public class StatisticalElasticSearchFacets extends InsecureElasticSearchFacets ElasticsearchAggregationData aggregationData = getElasticsearchAggregationData(); if (aggregationData == null || aggregationData.getNumberOfFacets() < numberOfFacets) { LOG.warn("Facets and Totalhit count are being retrieved by calling Elasticsearch"); - topChildren = super.getElasticSearchFacets(numberOfFacets); + topChildren = super.getElasticSearchFacets(indexDefinition, numberOfFacets); ElasticsearchSearcherModel elasticsearchSearcherModel = new ElasticsearchSearcherModel.ElasticsearchSearcherModelBuilder() .withQuery(getQuery()) .withBatchSize(ElasticsearchConstants.ELASTICSEARCH_QUERY_BATCH_SIZE) @@ -79,7 +82,8 @@ public class StatisticalElasticSearchFacets extends InsecureElasticSearchFacets // instead of statistical count. if (hitCount < sampleSize) { LOG.debug("SampleSize: {} is greater than hitcount: {}, Getting secure facet count", sampleSize, hitCount); - return new SecureElasticSearchFacets(getSearcher(), getQuery(), getPlan()).getElasticSearchFacets(numberOfFacets); + return new SecureElasticSearchFacets(getSearcher(), getQuery(), getPlan()).getElasticSearchFacets(indexDefinition, + numberOfFacets); } long randomSeed = secureFacetConfiguration.getRandomSeed(); Iterator docIterator = getMatchingDocIterator(getSearcher(), getQuery()); diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java index 939aaabd7a..86ab49e75d 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java @@ -25,8 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -35,10 +33,7 @@ import java.util.Map; class ElasticsearchDocument { private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchDocument.class); - // id should only be useful for logging (at least as of now) private final String path; - - private final String id; private final List fulltext; private final List suggest; private final List notNullProps; @@ -47,13 +42,6 @@ class ElasticsearchDocument { ElasticsearchDocument(String path) { this.path = path; - String id = null; - try { - id = pathToId(path); - } catch (UnsupportedEncodingException e) { - LOG.warn("Couldn't encode {} as ES id", path); - } - this.id = id; this.fulltext = new ArrayList<>(); this.suggest = new ArrayList<>(); this.notNullProps = new ArrayList<>(); @@ -93,21 +81,17 @@ class ElasticsearchDocument { String parPath = PathUtils.getParentPath(path); int depth = PathUtils.getDepth(path); - // TODO: remember that mapping must be configured with - // https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-pathhierarchy-tokenizer.html addProperty(FieldNames.ANCESTORS, parPath); addProperty(FieldNames.PATH_DEPTH, depth); } - String getId() { - return id; - } - public String build() { - String ret = null; + String ret; try { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); + { + builder.field(FieldNames.PATH, path); if (fulltext.size() > 0) { builder.field(FieldNames.FULLTEXT, fulltext); } @@ -123,14 +107,15 @@ class ElasticsearchDocument { for (Map.Entry prop : properties.entrySet()) { builder.field(prop.getKey(), prop.getValue()); } + } builder.endObject(); ret = Strings.toString(builder); } catch (IOException e) { - LOG.error("Error serializing document - id: {}, properties: {}, fulltext: {}, suggest: {}, " + + LOG.error("Error serializing document - path: {}, properties: {}, fulltext: {}, suggest: {}, " + "notNullProps: {}, nullProps: {}", - path, properties, fulltext, suggest, notNullProps, nullProps, - e); + path, properties, fulltext, suggest, notNullProps, nullProps, e); + ret = null; } return ret; @@ -140,8 +125,4 @@ class ElasticsearchDocument { public String toString() { return build(); } - - public static String pathToId(String path) throws UnsupportedEncodingException { - return URLEncoder.encode(path, "UTF-8"); - } } diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocumentMaker.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocumentMaker.java index adadb25750..6558ffeda6 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocumentMaker.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocumentMaker.java @@ -48,10 +48,9 @@ class ElasticsearchDocumentMaker extends FulltextDocumentMaker>> multiTypesFields = indexDefinition.getPropertiesByName() + .entrySet() + .stream() + .filter(e -> e.getValue().size() > 1) + .filter(e -> e.getValue().stream().map(PropertyDefinition::getType).distinct().count() > 1) + .collect(Collectors.toList()); + + if (!multiTypesFields.isEmpty()) { + String fields = multiTypesFields.stream().map(Map.Entry::getKey).collect(Collectors.joining(", ", "[", "]")); + throw new IllegalStateException(indexDefinition.getIndexPath() + " has properties with the same name and " + + "different types " + fields); + } + + for (Map.Entry> entry : indexDefinition.getPropertiesByName().entrySet()) { + final String name = entry.getKey(); + final List propertyDefinitions = entry.getValue(); + + Type type = Type.fromTag(propertyDefinitions.get(0).getType(), false); + mappingBuilder.startObject(name); + { + // https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html + if (Type.BINARY.equals(type)) { + mappingBuilder.field("type", "binary"); + } else if (Type.LONG.equals(type)) { + mappingBuilder.field("type", "long"); + } else if (Type.DOUBLE.equals(type) || Type.DECIMAL.equals(type)) { + mappingBuilder.field("type", "double"); + } else if (Type.DATE.equals(type)) { + mappingBuilder.field("type", "date"); + } else if (Type.BOOLEAN.equals(type)) { + mappingBuilder.field("type", "boolean"); + } else { + if (indexDefinition.isAnalyzed(propertyDefinitions)) { + mappingBuilder.field("type", "text"); + // always add keyword for sorting / faceting as sub-field + mappingBuilder.startObject("fields"); + { + mappingBuilder.startObject("keyword") + .field("type", "keyword") + .endObject(); + } + mappingBuilder.endObject(); + } else { + // always add keyword for sorting / faceting + mappingBuilder.field("type", "keyword"); + } + } + } + mappingBuilder.endObject(); + } + } +} diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java index 59215d4cf8..c55c6478f8 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java @@ -18,8 +18,8 @@ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.index; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection; import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; -import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriter; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; @@ -37,13 +37,11 @@ import org.elasticsearch.client.IndicesClient; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.TestOnly; @@ -51,12 +49,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.index.ElasticsearchDocument.pathToId; import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -66,8 +69,19 @@ class ElasticsearchIndexWriter implements FulltextIndexWriter updatesMap = new ConcurrentHashMap<>(); private final BulkProcessor bulkProcessor; - private Optional indexUpdated = Optional.empty(); ElasticsearchIndexWriter(@NotNull ElasticsearchConnection elasticsearchConnection, @NotNull ElasticsearchIndexDefinition indexDefinition) { @@ -99,70 +113,76 @@ class ElasticsearchIndexWriter implements FulltextIndexWriter timeoutMillis) { - // indexUpdate was not set till now, return false - LOG.trace("Timed out waiting for the bulk processor response. Returning indexUpdated = false"); - return false; - } else { + // de-register main controller + final int phase = phaser.arriveAndDeregister(); + try { - LOG.trace("Waiting for afterBulk response..."); - Thread.sleep(100); - } catch (InterruptedException ex) { - // - } + phaser.awaitAdvanceInterruptibly(phase, indexDefinition.bulkFlushIntervalMs * 5, TimeUnit.MILLISECONDS); + } catch (InterruptedException | TimeoutException e) { + LOG.error("Error waiting for bulk requests to return", e); } + + if (LOG.isTraceEnabled()) { + LOG.trace("Bulk identifier -> update status = {}", updatesMap); } - LOG.trace("Returning indexUpdated = {}", indexUpdated.get()); - return indexUpdated.get(); + return updatesMap.containsValue(Boolean.TRUE); } - // TODO: we need to check if the index already exists and in that case we have to figure out if there are - // "breaking changes" in the index definition protected void provisionIndex() throws IOException { + // check if index already exists + boolean exists = elasticsearchConnection.getClient().indices().exists( + new GetIndexRequest(indexDefinition.getRemoteIndexName()), RequestOptions.DEFAULT + ); + if (exists) { + LOG.info("Index {} already exists. Skip index provision", indexDefinition.getRemoteIndexName()); + return; + } - IndicesClient indicesClient = elasticsearchConnection.getClient().indices(); + final IndicesClient indicesClient = elasticsearchConnection.getClient().indices(); final String indexName = indexDefinition.getRemoteIndexName(); - CreateIndexRequest createIndexRequest = constructCreateIndexRequest(indexName); - String requestMsg = Strings.toString(createIndexRequest.toXContent(jsonBuilder(), EMPTY_PARAMS)); - CreateIndexResponse response = indicesClient.create(createIndexRequest, RequestOptions.DEFAULT); + // create the new index + final CreateIndexRequest request = ElasticsearchIndexHelper.createIndexRequest(indexDefinition); + try { + if (LOG.isDebugEnabled()) { + final String requestMsg = Strings.toString(request.toXContent(jsonBuilder(), EMPTY_PARAMS)); + LOG.debug("Creating Index with request {}", requestMsg); + } + CreateIndexResponse response = indicesClient.create(request, RequestOptions.DEFAULT); + LOG.info("Updated settings for index {}. Response acknowledged: {}", + indexDefinition.getRemoteIndexName(), response.isAcknowledged()); checkResponseAcknowledgement(response, "Create index call not acknowledged for index " + indexName); + } catch (ElasticsearchStatusException ese) { + // We already check index existence as first thing in this method, if we get here it means we have got into + // a conflict (eg: multiple cluster nodes provision concurrently). + // Elasticsearch does not have a CREATE IF NOT EXIST, need to inspect exception + // https://github.com/elastic/elasticsearch/issues/19862 + if (ese.status().getStatus() == 400 && ese.getDetailedMessage().contains("resource_already_exists_exception")) { + LOG.warn("Index {} already exists. Ignoring error", indexName); + } else throw ese; + } - LOG.info("Updated settings for index {} = {}. Response acknowledged: {}", - indexDefinition.getRemoteIndexAlias(), requestMsg, response.isAcknowledged()); - - + // update the mapping GetAliasesRequest getAliasesRequest = new GetAliasesRequest(indexDefinition.getRemoteIndexAlias()); GetAliasesResponse aliasesResponse = indicesClient.getAlias(getAliasesRequest, RequestOptions.DEFAULT); Map> aliases = aliasesResponse.getAliases(); @@ -180,6 +200,8 @@ class ElasticsearchIndexWriter implements FulltextIndexWriter {}", executionId, bulkRequest.getDescription()); if (LOG.isTraceEnabled()) { LOG.trace("Bulk Requests: \n{}", bulkRequest.requests() @@ -279,29 +259,41 @@ class ElasticsearchIndexWriter implements FulltextIndexWriter + * Mapping _id field + */ + private static String idFromPath(@NotNull String path) { + byte[] pathBytes = path.getBytes(StandardCharsets.UTF_8); + if (pathBytes.length > 512) { + try { + return new String(MessageDigest.getInstance("SHA-256").digest(pathBytes)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } } + return path; } } diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java index 1c06a5d520..51bd925ef5 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java @@ -90,7 +90,9 @@ class ElasticsearchIndex extends FulltextIndex { @Override protected String getFulltextRequestString(IndexPlan plan, IndexNode indexNode) { - return Strings.toString(ElasticsearchResultRowIterator.getESQuery(plan, getPlanResult(plan))); + return Strings.toString(new ElasticsearchResultRowIterator(plan.getFilter(), getPlanResult(plan), plan, + acquireIndexNode(plan), FulltextIndex::shouldInclude, getEstimator(plan.getPlanName())) + .getESQuery(plan, getPlanResult(plan))); } @Override diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java index 05429984b6..f173d29758 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java @@ -53,8 +53,6 @@ import org.slf4j.LoggerFactory; import javax.jcr.PropertyType; import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; @@ -68,7 +66,6 @@ import java.util.stream.StreamSupport; import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES; import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE; -import static org.apache.jackrabbit.oak.api.Type.STRING; import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot; import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newAncestorQuery; @@ -158,9 +155,10 @@ class ElasticsearchResultRowIterator implements Iterator aggregationBuilders = ElasticsearchAggregationBuilderUtil - .getAggregators(plan, numberOfFacets); + .getAggregators(plan, indexDefinition, numberOfFacets); ElasticsearchSearcherModel elasticsearchSearcherModel = new ElasticsearchSearcherModel.ElasticsearchSearcherModelBuilder() .withQuery(query) @@ -236,10 +234,9 @@ class ElasticsearchResultRowIterator implements Iterator sourceMap = hit.getSourceAsMap(); + String path = (String) sourceMap.get(FieldNames.PATH); if (path != null) { if ("".equals(path)) { path = "/"; @@ -276,7 +273,7 @@ class ElasticsearchResultRowIterator implements Iterator qs = new ArrayList<>(); Filter filter = plan.getFilter(); FullTextExpression ft = filter.getFullTextConstraint(); @@ -504,13 +501,12 @@ class ElasticsearchResultRowIterator implements Iterator qs, + private void addNonFullTextConstraints(List qs, IndexPlan plan, PlanResult planResult) { final BiPredicate, String> any = (iterable, value) -> StreamSupport.stream(iterable.spliterator(), false).anyMatch(value::equals); Filter filter = plan.getFilter(); - IndexDefinition defn = planResult.indexDefinition; if (!filter.matchesAllTypes()) { addNodeTypeConstraints(planResult.indexingRule, qs, filter); } @@ -518,20 +514,15 @@ class ElasticsearchResultRowIterator implements Iterator parse(value.getValue(Type.DATE)).getTime()); + in = newPropertyRestrictionQuery(field, pr, value -> parse(value.getValue(Type.DATE)).getTime()); break; } case PropertyType.DOUBLE: { - in = newPropertyRestrictionQuery(propertyName, false, pr, - value -> value.getValue(Type.DOUBLE)); + in = newPropertyRestrictionQuery(field, pr, value -> value.getValue(Type.DOUBLE)); break; } case PropertyType.LONG: { - in = newPropertyRestrictionQuery(propertyName, false, pr, - value -> value.getValue(Type.LONG)); + in = newPropertyRestrictionQuery(field, pr, value -> value.getValue(Type.LONG)); break; } default: { if (pr.isLike) { - return createLikeQuery(propertyName, pr.first.getValue(STRING)); + return createLikeQuery(propertyName, pr.first.getValue(Type.STRING)); } //TODO Confirm that all other types can be treated as string - in = newPropertyRestrictionQuery(propertyName, true, pr, - value -> value.getValue(Type.STRING)); + in = newPropertyRestrictionQuery(field, pr, value -> value.getValue(Type.STRING)); } } @@ -734,10 +723,6 @@ class ElasticsearchResultRowIterator implements Iterator> cachedResults = new HashMap<>(); @@ -750,7 +735,7 @@ class ElasticsearchResultRowIterator implements Iterator getFacets(int numberOfFacets, String columnName) throws IOException { String facetProp = FulltextIndex.parseFacetField(columnName); if (cachedResults.get(facetProp) == null) { - cachedResults = elasticsearchFacets.getElasticSearchFacets(numberOfFacets); + cachedResults = elasticsearchFacets.getElasticSearchFacets(indexNode.getDefinition(), numberOfFacets); } return cachedResults.get(facetProp); } diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchSearcher.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchSearcher.java index 3eb9ebbc23..3c9f515314 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchSearcher.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchSearcher.java @@ -47,8 +47,7 @@ public class ElasticsearchSearcher { public SearchResponse search(QueryBuilder query, int batchSize) throws IOException { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .query(query) - .fetchSource(false) - .storedField(FieldNames.PATH) + .fetchSource(FieldNames.PATH, null) .size(batchSize); SearchRequest request = new SearchRequest(indexNode.getDefinition().getRemoteIndexAlias()) diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/ElasticsearchAggregationBuilderUtil.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/ElasticsearchAggregationBuilderUtil.java index c745ba779e..70415abbac 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/ElasticsearchAggregationBuilderUtil.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/ElasticsearchAggregationBuilderUtil.java @@ -17,6 +17,7 @@ package org.apache.jackrabbit.oak.plugins.index.elasticsearch.util; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex; import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.jackrabbit.oak.spi.query.QueryConstants; @@ -33,7 +34,7 @@ public final class ElasticsearchAggregationBuilderUtil { private ElasticsearchAggregationBuilderUtil() { } - public static List getAggregators(QueryIndex.IndexPlan plan, int numberOfFacets) { + public static List getAggregators(QueryIndex.IndexPlan plan, ElasticsearchIndexDefinition indexDefinition, int numberOfFacets) { List termsAggregationBuilders = new LinkedList<>(); Collection propertyRestrictions = plan.getFilter().getPropertyRestrictions(); for (Filter.PropertyRestriction propertyRestriction : propertyRestrictions) { @@ -41,13 +42,14 @@ public final class ElasticsearchAggregationBuilderUtil { if (QueryConstants.REP_FACET.equals(name)) { String value = propertyRestriction.first.getValue(Type.STRING); String facetProp = FulltextIndex.parseFacetField(value); - termsAggregationBuilders.add(AggregationBuilders.terms(facetProp).field(keywordFieldName(facetProp)).size(numberOfFacets)); + termsAggregationBuilders.add( + AggregationBuilders + .terms(facetProp) + .field(indexDefinition.getElasticKeyword(facetProp)) + .size(numberOfFacets) + ); } } return termsAggregationBuilders; } - - private static String keywordFieldName(String propName) { - return propName + "." + "keyword"; - } } diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/SearchSourceBuilderUtil.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/SearchSourceBuilderUtil.java index a9ab309a8c..7c53974cbb 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/SearchSourceBuilderUtil.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/SearchSourceBuilderUtil.java @@ -25,8 +25,7 @@ public class SearchSourceBuilderUtil { public static SearchSourceBuilder createSearchSourceBuilder(ElasticsearchSearcherModel elasticsearchSearcherModel) { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .query(elasticsearchSearcherModel.getQueryBuilder()) - .fetchSource(elasticsearchSearcherModel.fetchSource()) - .storedField(elasticsearchSearcherModel.getStoredField()) + .fetchSource(elasticsearchSearcherModel.getStoredField(), null) .size(elasticsearchSearcherModel.getBatchSize()) .from(elasticsearchSearcherModel.getFrom()); diff --git oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/TermQueryBuilderFactory.java oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/TermQueryBuilderFactory.java index 67fa899a73..770d435999 100644 --- oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/TermQueryBuilderFactory.java +++ oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/TermQueryBuilderFactory.java @@ -67,11 +67,11 @@ public class TermQueryBuilderFactory { } public static PrefixQueryBuilder newPrefixQuery(String field, @NotNull String value) { - return prefixQuery(keywordFieldName(field), value); + return prefixQuery(field, value); } public static WildcardQueryBuilder newWildcardQuery(String field, @NotNull String value) { - return wildcardQuery(keywordFieldName(field), value); + return wildcardQuery(field, value); } public static TermQueryBuilder newPathQuery(String path) { @@ -96,11 +96,11 @@ public class TermQueryBuilderFactory { } public static TermQueryBuilder newNodeTypeQuery(String type) { - return termQuery(keywordFieldName(JCR_PRIMARYTYPE), type); + return termQuery(JCR_PRIMARYTYPE, type); } public static TermQueryBuilder newMixinTypeQuery(String type) { - return termQuery(keywordFieldName(JCR_MIXINTYPES), type); + return termQuery(JCR_MIXINTYPES, type); } public static TermQueryBuilder newNotNullPropQuery(String propName) { @@ -126,12 +126,9 @@ public class TermQueryBuilderFactory { return bq; } - public static QueryBuilder newPropertyRestrictionQuery(String propertyName, boolean isString, + public static QueryBuilder newPropertyRestrictionQuery(String propertyName, Filter.PropertyRestriction pr, Function propToObj) { - if (isString) { - propertyName = keywordFieldName(propertyName); - } R first = pr.first != null ? propToObj.apply(pr.first) : null; R last = pr.last != null ? propToObj.apply(pr.last) : null; @@ -166,9 +163,4 @@ public class TermQueryBuilderFactory { } return path; } - - // As per https://www.elastic.co/blog/strings-are-dead-long-live-strings - private static String keywordFieldName(String propName) { - return propName + "." + "keyword"; - } } \ No newline at end of file diff --git oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchAbstractQueryTest.java oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchAbstractQueryTest.java index 298fc37e8d..8bbc8273ee 100644 --- oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchAbstractQueryTest.java +++ oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchAbstractQueryTest.java @@ -20,7 +20,7 @@ import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.InitialContent; import org.apache.jackrabbit.oak.Oak; import org.apache.jackrabbit.oak.api.ContentRepository; -import org.apache.jackrabbit.oak.commons.PerfLogger; +import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; import org.apache.jackrabbit.oak.plugins.index.TrackingCorruptIndexHandler; @@ -37,6 +37,9 @@ import org.apache.jackrabbit.oak.query.AbstractQueryTest; import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.client.indices.GetIndexRequest; import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.ClassRule; @@ -44,7 +47,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.net.URI; import java.util.concurrent.TimeUnit; import static com.google.common.collect.Lists.newArrayList; @@ -56,10 +58,6 @@ public abstract class ElasticsearchAbstractQueryTest extends AbstractQueryTest { protected static final Logger LOG = LoggerFactory.getLogger(ElasticsearchAbstractQueryTest.class); - protected static final PerfLogger PERF_LOGGER = - new PerfLogger(LoggerFactory.getLogger(ElasticsearchAbstractQueryTest.class.getName() + ".perf")); - - // Set this connection string as // ://:?key_id=<>,key_secret=<> // key_id and key_secret are optional in case the ES server @@ -72,11 +70,9 @@ public abstract class ElasticsearchAbstractQueryTest extends AbstractQueryTest { // This can be used by the extending classes to trigger the async index update as per need (not having to wait for async indexing cycle) protected AsyncIndexUpdate asyncIndexUpdate; protected long INDEX_CORRUPT_INTERVAL_IN_MILLIS = 100; - protected ElasticsearchIndexEditorProvider editorProvider; protected NodeStore nodeStore; protected int DEFAULT_ASYNC_INDEXING_TIME_IN_SECONDS = 5; - @ClassRule public static ElasticsearchConnectionRule elasticRule = new ElasticsearchConnectionRule(elasticConnectionString); @@ -131,12 +127,11 @@ public abstract class ElasticsearchAbstractQueryTest extends AbstractQueryTest { } protected Oak addAsyncIndexingLanesToOak(Oak oak) { - // Override this in extending clases to configure different + // Override this in extending classes to configure different // indexing lanes with different time limits. return oak.withAsyncIndexing("async", DEFAULT_ASYNC_INDEXING_TIME_IN_SECONDS); } - @Override protected ContentRepository createRepository() { @@ -156,7 +151,6 @@ public abstract class ElasticsearchAbstractQueryTest extends AbstractQueryTest { trackingCorruptIndexHandler.setCorruptInterval(INDEX_CORRUPT_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS); asyncIndexUpdate.setCorruptIndexHandler(trackingCorruptIndexHandler); - Oak oak = new Oak(nodeStore) .with(getInitialContent()) .with(new OpenSecurityProvider()) @@ -171,7 +165,6 @@ public abstract class ElasticsearchAbstractQueryTest extends AbstractQueryTest { return oak.createContentRepository(); } - protected static void assertEventually(Runnable r) { ElasticsearchTestUtils.assertEventually(r, BULK_FLUSH_INTERVAL_MS_DEFAULT * 5); } @@ -188,8 +181,8 @@ public abstract class ElasticsearchAbstractQueryTest extends AbstractQueryTest { return builder; } - protected void setIndex(String idxName, IndexDefinitionBuilder builder) { - builder.build(root.getTree("/").addChild(INDEX_DEFINITIONS_NAME).addChild(idxName)); + protected Tree setIndex(String idxName, IndexDefinitionBuilder builder) { + return builder.build(root.getTree("/").addChild(INDEX_DEFINITIONS_NAME).addChild(idxName)); } protected String explain(String query) { @@ -206,4 +199,36 @@ public abstract class ElasticsearchAbstractQueryTest extends AbstractQueryTest { setTraversalEnabled(false); } + // Utility methods accessing directly Elasticsearch + + protected boolean exists(Tree index) { + ElasticsearchIndexDefinition esIdxDef = getElasticsearchIndexDefinition(index); + + try { + return esConnection.getClient().indices() + .exists(new GetIndexRequest(esIdxDef.getRemoteIndexAlias()), RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + protected long countDocuments(Tree index) { + ElasticsearchIndexDefinition esIdxDef = getElasticsearchIndexDefinition(index); + + CountRequest request = new CountRequest(esIdxDef.getRemoteIndexAlias()); + try { + return esConnection.getClient().count(request, RequestOptions.DEFAULT).getCount(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private ElasticsearchIndexDefinition getElasticsearchIndexDefinition(Tree index) { + return new ElasticsearchIndexDefinition( + nodeStore.getRoot(), + nodeStore.getRoot().getChildNode(INDEX_DEFINITIONS_NAME).getChildNode(index.getName()), + index.getPath(), + esConnection.getIndexPrefix()); + } + } diff --git oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchContentTest.java oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchContentTest.java new file mode 100644 index 0000000000..aeb7e5d1ef --- /dev/null +++ oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchContentTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.elasticsearch; + +import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Random; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertTrue; + +public class ElasticsearchContentTest extends ElasticsearchAbstractQueryTest { + + @Test + public void indexWithAnalyzedProperty() throws Exception { + IndexDefinitionBuilder builder = createIndex("a").noAsync(); + builder.indexRule("nt:base").property("a").analyzed(); + String testId = UUID.randomUUID().toString(); + Tree index = setIndex(testId, builder); + root.commit(); + + assertTrue(exists(index)); + assertThat(0L, equalTo(countDocuments(index))); + + Tree content = root.getTree("/").addChild(testId); + content.addChild("indexed").setProperty("a", "foo"); + content.addChild("not-indexed").setProperty("b", "foo"); + root.commit(); + + assertEventually(() -> assertThat(countDocuments(index), equalTo(1L))); + + content.getChild("indexed").remove(); + root.commit(); + + assertEventually(() -> assertThat(countDocuments(index), equalTo(0L))); + + // TODO: should the index be deleted when the definition gets removed? + //index.remove(); + //root.commit(); + + //assertFalse(exists(index)); + } + + @Test + @Ignore("this test fails because of a bug with nodeScopeIndex (every node gets indexed in an empty doc)") + public void indexWithAnalyzedNodeScopeIndexProperty() throws Exception { + IndexDefinitionBuilder builder = createIndex("a").noAsync(); + builder.indexRule("nt:base").property("a").analyzed().nodeScopeIndex(); + String testId = UUID.randomUUID().toString(); + Tree index = setIndex(testId, builder); + root.commit(); + + assertThat(0L, equalTo(countDocuments(index))); + + Tree content = root.getTree("/").addChild(testId); + content.addChild("indexed").setProperty("a", "foo"); + content.addChild("not-indexed").setProperty("b", "foo"); + root.commit(); + + assertEventually(() -> assertThat(countDocuments(index), equalTo(1L))); + } + + @Test + public void indexContentWithLongPath() throws Exception { + IndexDefinitionBuilder builder = createIndex("a").noAsync(); + builder.indexRule("nt:base").property("a").analyzed(); + String testId = UUID.randomUUID().toString(); + Tree index = setIndex(testId, builder); + root.commit(); + + assertTrue(exists(index)); + assertThat(0L, equalTo(countDocuments(index))); + + int leftLimit = 48; // ' ' (space) + int rightLimit = 122; // char '~' + int targetStringLength = 1024; + final Random random = new Random(42); + + String generatedPath = random.ints(leftLimit, rightLimit + 1) + .limit(targetStringLength) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + + Tree content = root.getTree("/").addChild(testId); + content.addChild(generatedPath).setProperty("a", "foo"); + root.commit(); + + assertEventually(() -> assertThat(countDocuments(index), equalTo(1L))); + } + + @Test + public void defineIndexTwice() throws Exception { + IndexDefinitionBuilder builder = createIndex("a").noAsync(); + String testId = UUID.randomUUID().toString(); + Tree index = setIndex(testId, builder); + root.commit(); + + assertTrue(exists(index)); + + builder = createIndex("a").noAsync(); + index = setIndex(testId, builder); + root.commit(); + } +} diff --git oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchFacetTest.java oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchFacetTest.java index 1c0536f56f..49ded1daba 100644 --- oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchFacetTest.java +++ oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchFacetTest.java @@ -83,15 +83,14 @@ public class ElasticsearchFacetTest { private QueryManager qe; Repository repository; private Node indexNode; - private static String TEST_INDEX = "testIndex"; + private static final String TEST_INDEX = "testIndex"; private static final int NUM_LEAF_NODES = STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT; private static final int NUM_LABELS = 4; private static final int NUM_LEAF_NODES_FOR_LARGE_DATASET = NUM_LEAF_NODES; private static final int NUM_LEAF_NODES_FOR_SMALL_DATASET = NUM_LEAF_NODES / (2 * NUM_LABELS); - private Map actualLabelCount = Maps.newHashMap(); - private Map actualAclLabelCount = Maps.newHashMap(); - private Map actualAclPar1LabelCount = Maps.newHashMap(); - + private final Map actualLabelCount = Maps.newHashMap(); + private final Map actualAclLabelCount = Maps.newHashMap(); + private final Map actualAclPar1LabelCount = Maps.newHashMap(); @Rule public final ElasticsearchContainer elastic = @@ -136,38 +135,21 @@ public class ElasticsearchFacetTest { Jcr jcr = new Jcr(oak); repository = jcr.createRepository(); - try { session = repository.login(new SimpleCredentials("admin", "admin".toCharArray()), null); - } catch (RepositoryException e) { - throw e; - } closer.register(session::logout); // we'd always query anonymously - Session anonSession = null; - try { - anonSession = repository.login(new GuestCredentials(), null); + Session anonSession = repository.login(new GuestCredentials(), null); anonSession.refresh(true); anonSession.save(); - } catch (RepositoryException e) { - throw e; - } closer.register(anonSession::logout); - try { qe = anonSession.getWorkspace().getQueryManager(); - } catch (RepositoryException e) { - throw e; - } } - private class IndexSkeleton { IndexDefinitionBuilder indexDefinitionBuilder; IndexDefinitionBuilder.IndexRule indexRule; - private IndexSkeleton() throws RepositoryException { - } - void initialize() { initialize(JcrConstants.NT_BASE); } @@ -185,7 +167,7 @@ public class ElasticsearchFacetTest { private void createIndex() throws RepositoryException { IndexSkeleton indexSkeleton = new IndexSkeleton(); indexSkeleton.initialize(); - indexSkeleton.indexDefinitionBuilder.noAsync().evaluatePathRestrictions(); + indexSkeleton.indexDefinitionBuilder.noAsync(); indexSkeleton.indexRule.property("cons").propertyIndex(); indexSkeleton.indexRule.property("foo").propertyIndex().getBuilderTree().setProperty(FACET_PROP, true, Type.BOOLEAN); indexSkeleton.indexRule.property("bar").propertyIndex().getBuilderTree().setProperty(FACET_PROP, true, Type.BOOLEAN); @@ -248,9 +230,7 @@ public class ElasticsearchFacetTest { @Test public void secureFacets() throws Exception { createDataset(NUM_LEAF_NODES_FOR_LARGE_DATASET); - assertEventually(() -> { - assertEquals(actualAclLabelCount, getFacets()); - }); + assertEventually(() -> assertEquals(actualAclLabelCount, getFacets())); } @Test @@ -260,9 +240,7 @@ public class ElasticsearchFacetTest { inaccessibleChild.setProperty("cons", "val"); inaccessibleChild.setProperty("foo", "l4"); session.save(); - assertEventually(() -> { - assertEquals(actualAclLabelCount, getFacets()); - }); + assertEventually(() -> assertEquals(actualAclLabelCount, getFacets())); } @Test @@ -272,9 +250,7 @@ public class ElasticsearchFacetTest { session.save(); createDataset(NUM_LEAF_NODES_FOR_LARGE_DATASET); - assertEventually(() -> { - assertEquals(actualLabelCount, getFacets()); - }); + assertEventually(() -> assertEquals(actualLabelCount, getFacets())); } @Test @@ -286,14 +262,12 @@ public class ElasticsearchFacetTest { createDataset(NUM_LEAF_NODES_FOR_LARGE_DATASET); - assertEventually(() -> { - assertEquals("Unexpected number of facets", actualAclLabelCount.size(), getFacets().size()); - }); + assertEventually(() -> assertEquals("Unexpected number of facets", actualAclLabelCount.size(), getFacets().size())); for (Map.Entry facet : actualAclLabelCount.entrySet()) { String facetLabel = facet.getKey(); assertEventually(() -> { - int facetCount = facetCount = getFacets().get(facetLabel); + int facetCount = getFacets().get(facetLabel); float ratio = ((float) facetCount) / facet.getValue(); assertTrue("Facet count for label: " + facetLabel + " is outside of 10% margin of error. " + "Expected: " + facet.getValue() + "; Got: " + facetCount + "; Ratio: " + ratio, @@ -311,21 +285,13 @@ public class ElasticsearchFacetTest { createDataset(NUM_LEAF_NODES_FOR_SMALL_DATASET); - assertEventually(() -> { - assertEquals("Unexpected number of facets", actualAclLabelCount.size(), getFacets().size()); - }); + assertEventually(() -> assertEquals("Unexpected number of facets", actualAclLabelCount.size(), getFacets().size())); // Since the hit count is less than sample size -> flow should have switched to secure facet count instead of statistical // and thus the count should be exactly equal - assertEventually(() -> { - assertEquals(actualAclLabelCount, getFacets()); - }); + assertEventually(() -> assertEquals(actualAclLabelCount, getFacets())); } - /* - Currently we are not adding path restrictions in elastic query and filtering paths later from results. - We will need path restrictions in elastic query itself to get right facet results. - */ @Test public void statisticalFacets_withHitCountSameAsSampleSize() throws Exception { Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" + FACETS, "nt:unstructured", session); @@ -335,6 +301,7 @@ public class ElasticsearchFacetTest { createDataset(NUM_LEAF_NODES_FOR_LARGE_DATASET); + assertEventually(() -> { Map facets = getFacets("/parent/par1"); assertEquals("Unexpected number of facets", actualAclPar1LabelCount.size(), facets.size()); @@ -346,6 +313,7 @@ public class ElasticsearchFacetTest { "Expected: " + facet.getValue() + "; Got: " + facetCount + "; Ratio: " + ratio, Math.abs(ratio - 1) < 0.1); } + }); } @Test @@ -386,9 +354,7 @@ public class ElasticsearchFacetTest { session.save(); createDataset(NUM_LEAF_NODES_FOR_LARGE_DATASET); qe = session.getWorkspace().getQueryManager(); - assertEventually(() -> { - assertEquals(actualLabelCount, getFacets()); - }); + assertEventually(() -> assertEquals(actualLabelCount, getFacets())); } @Test @@ -438,7 +404,7 @@ public class ElasticsearchFacetTest { pathCons = " AND ISDESCENDANTNODE('" + path + "')"; } String query = "SELECT [rep:facet(foo)], [rep:facet(bar)] FROM [nt:base] WHERE [cons] = 'val'" + pathCons; - Query q = null; + Query q; QueryResult queryResult; try { q = qe.createQuery(query, Query.JCR_SQL2); diff --git oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchFullTextAsyncTest.java oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchFullTextAsyncTest.java index b1b769f47d..a732d1af7c 100644 --- oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchFullTextAsyncTest.java +++ oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchFullTextAsyncTest.java @@ -16,7 +16,6 @@ */ package org.apache.jackrabbit.oak.plugins.index.elasticsearch; - import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder; import org.junit.Test; diff --git oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchPropertyIndexTest.java oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchPropertyIndexTest.java index 3e7f8d50f9..4793898c2b 100644 --- oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchPropertyIndexTest.java +++ oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchPropertyIndexTest.java @@ -19,8 +19,6 @@ package org.apache.jackrabbit.oak.plugins.index.elasticsearch; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; @@ -96,9 +94,8 @@ public class ElasticsearchPropertyIndexTest extends ElasticsearchAbstractQueryTe //make index IndexDefinitionBuilder builder = createIndex(); builder.includedPaths("/test") - .evaluatePathRestrictions() .indexRule("nt:base") - .property("nodeName", PROPDEF_PROP_NODE_NAME).propertyIndex(); + .property("nodeName", PROPDEF_PROP_NODE_NAME); setIndex("test1", builder); root.commit(); diff --git oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexHelperTest.java oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexHelperTest.java new file mode 100644 index 0000000000..54fbd48846 --- /dev/null +++ oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexHelperTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.elasticsearch.index; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.ElasticsearchIndexDefinitionBuilder; +import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class ElasticsearchIndexHelperTest { + + @Test + public void multiRulesWithSamePropertyNames() throws IOException { + IndexDefinitionBuilder builder = new ElasticsearchIndexDefinitionBuilder(); + IndexDefinitionBuilder.IndexRule indexRuleA = builder.indexRule("typeA"); + indexRuleA.property("foo").type("String"); + IndexDefinitionBuilder.IndexRule indexRuleB = builder.indexRule("typeB"); + indexRuleB.property("foo").type("String").analyzed(); + NodeState nodeState = builder.build(); + + ElasticsearchIndexDefinition definition = + new ElasticsearchIndexDefinition(nodeState, nodeState, "path", "prefix"); + + CreateIndexRequest request = ElasticsearchIndexHelper.createIndexRequest(definition); + + ObjectMapper mapper = new ObjectMapper(); + Map jsonMap = mapper.readValue(request.mappings().streamInput(), Map.class); + + Map fooMapping = (Map) ((Map) jsonMap.get("properties")).get("foo"); + assertThat(fooMapping.get("type"), is("text")); + Map fooKeywordMapping = (Map) ((Map) fooMapping.get("fields")).get("keyword"); + assertThat(fooKeywordMapping.get("type"), is("keyword")); + } + + @Test(expected = IllegalStateException.class) + public void multiRulesWithSamePropertyNamesDifferentTypes() throws IOException { + IndexDefinitionBuilder builder = new ElasticsearchIndexDefinitionBuilder(); + IndexDefinitionBuilder.IndexRule indexRuleA = builder.indexRule("typeA"); + indexRuleA.property("foo").type("String"); + IndexDefinitionBuilder.IndexRule indexRuleB = builder.indexRule("typeB"); + indexRuleB.property("foo").type("Boolean"); + NodeState nodeState = builder.build(); + + ElasticsearchIndexDefinition definition = + new ElasticsearchIndexDefinition(nodeState, nodeState, "path", "prefix"); + + ElasticsearchIndexHelper.createIndexRequest(definition); + } + +} diff --git oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriterTest.java oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriterTest.java index 838ac0d026..9fa9bb8927 100644 --- oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriterTest.java +++ oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriterTest.java @@ -27,9 +27,11 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import java.io.IOException; -import java.net.URLEncoder; +import java.util.Random; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.number.OrderingComparison.lessThan; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -56,31 +58,31 @@ public class ElasticsearchIndexWriterTest { } @Test - public void singleUpdateDocument() throws IOException { + public void singleUpdateDocument() { indexWriter.updateDocument("/foo", new ElasticsearchDocument("/foo")); ArgumentCaptor acIndexRequest = ArgumentCaptor.forClass(IndexRequest.class); verify(bulkProcessorMock).add(acIndexRequest.capture()); IndexRequest request = acIndexRequest.getValue(); - assertEquals(request.index(), "test-index"); - assertEquals(request.id(), URLEncoder.encode("/foo", "UTF-8")); + assertEquals("test-index", request.index()); + assertEquals("/foo", request.id()); } @Test - public void singleDeleteDocument() throws IOException { + public void singleDeleteDocument() { indexWriter.deleteDocuments("/bar"); ArgumentCaptor acDeleteRequest = ArgumentCaptor.forClass(DeleteRequest.class); verify(bulkProcessorMock).add(acDeleteRequest.capture()); DeleteRequest request = acDeleteRequest.getValue(); - assertEquals(request.index(), "test-index"); - assertEquals(request.id(), URLEncoder.encode("/bar", "UTF-8")); + assertEquals("test-index", request.index()); + assertEquals("/bar", request.id()); } @Test - public void multiRequests() throws IOException { + public void multiRequests() { indexWriter.updateDocument("/foo", new ElasticsearchDocument("/foo")); indexWriter.updateDocument("/bar", new ElasticsearchDocument("/bar")); indexWriter.deleteDocuments("/foo"); @@ -93,7 +95,29 @@ public class ElasticsearchIndexWriterTest { } @Test - public void closeBulkProcessor() throws IOException { + public void longDocumentPath() { + int leftLimit = 48; // '0' + int rightLimit = 122; // char '~' + int targetStringLength = 1024; + final Random random = new Random(42); + + String generatedPath = random.ints(leftLimit, rightLimit + 1) + .limit(targetStringLength) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + + indexWriter.updateDocument(generatedPath, new ElasticsearchDocument(generatedPath)); + + ArgumentCaptor acIndexRequest = ArgumentCaptor.forClass(IndexRequest.class); + verify(bulkProcessorMock).add(acIndexRequest.capture()); + + IndexRequest request = acIndexRequest.getValue(); + assertThat(request.id(), not(generatedPath)); + assertThat(request.id().length(), lessThan(513)); + } + + @Test + public void closeBulkProcessor() { indexWriter.close(System.currentTimeMillis()); verify(bulkProcessorMock).close(); } diff --git oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java index 81c8927557..e2b966d36c 100644 --- oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java +++ oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java @@ -1029,6 +1029,18 @@ public class IndexDefinition implements Aggregate.AggregateMapper { return baseNodeType; } + /** + * Returns all the configured {@code PropertyDefinition}s for this {@code IndexRule}. + * + * In case of a pure nodetype index we just return primaryType and mixins. + * + * @return an {@code Iterable} of {@code PropertyDefinition}s. + * @see IndexDefinition#isPureNodeTypeIndex() + */ + public Iterable getProperties() { + return propConfigs.values(); + } + public List getNullCheckEnabledProperties() { return nullCheckEnabledProperties; }