From df13e3b4a623defdf66a9816a75932359e0a6550 Mon Sep 17 00:00:00 2001 From: Vikas Saurabh Date: Mon, 3 Dec 2018 01:26:44 +0530 Subject: [PATCH 2/2] OAK-7929: Incorrect Facet Count With Large Dataset and ACLs * Allow secure and sampleSize under /facets * These can be overridden by JVM command line param * Inject seed into index definition during indexing cycle * Rename FilteredFacetCounts to SecureFacetCounts to serve for secure configuration * StatisticalSortedSetDocValueFacetCounts would do random sampling of results wrt what ratio are accessible. Use this ratio to extrapolate facet counts returned by index * Add TemporarySystemProperty rule to ease writing tests which set system property --- .../junit/TemporarySystemProperty.java | 39 ++++ .../index/lucene/LucenePropertyIndex.java | 4 +- .../index/lucene/util/FacetHelper.java | 27 ++- ... SecureSortedSetDocValuesFacetCounts.java} | 27 ++- ...tisticalSortedSetDocValuesFacetCounts.java | 190 ++++++++++++++++ .../index/lucene/LucenePropertyIndexTest.java | 128 ++++++++++- oak-search/pom.xml | 7 + .../index/search/FulltextIndexConstants.java | 16 ++ .../plugins/index/search/IndexDefinition.java | 132 +++++++++-- .../editor/FulltextIndexEditorContext.java | 38 ++++ .../IndexDefinitionFacetConfigTest.java | 211 ++++++++++++++++++ 11 files changed, 771 insertions(+), 48 deletions(-) create mode 100644 oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java rename oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/{FilteredSortedSetDocValuesFacetCounts.java => SecureSortedSetDocValuesFacetCounts.java} (83%) create mode 100644 oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java create mode 100644 oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java diff --git a/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java new file mode 100644 index 0000000000..604263dfcd --- /dev/null +++ b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.commons.junit; + +import org.junit.rules.ExternalResource; + +import java.util.Properties; + +public class TemporarySystemProperty extends ExternalResource { + + Properties oldProps = new Properties(); + + @Override + protected void before() { + oldProps.putAll(System.getProperties()); + } + + @Override + protected void after() { + System.setProperties(oldProps); + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java index bb5134c413..4a2dab9f94 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java @@ -120,7 +120,6 @@ import org.apache.lucene.search.highlight.TextFragment; import org.apache.lucene.search.postingshighlight.PostingsHighlighter; import org.apache.lucene.search.spell.SuggestWord; import org.apache.lucene.search.suggest.Lookup; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -129,7 +128,6 @@ import org.slf4j.LoggerFactory; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Predicates.in; import static com.google.common.base.Predicates.notNull; import static com.google.common.collect.Lists.newArrayListWithCapacity; import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES; @@ -353,7 +351,7 @@ public class LucenePropertyIndex extends FulltextIndex { long f = PERF_LOGGER.start(); if (facetProvider == null) { facetProvider = new LuceneFacetProvider( - FacetHelper.getFacets(searcher, query, docs, plan, indexNode.getDefinition().isSecureFacets()) + FacetHelper.getFacets(searcher, query, plan, indexNode.getDefinition().getSecureFacetConfiguration()) ); PERF_LOGGER.end(f, -1, "facets retrieved"); } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java index a5ae8bfe6b..f745e57bc6 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration; import org.apache.jackrabbit.oak.spi.query.QueryConstants; import org.apache.jackrabbit.oak.spi.query.QueryIndex; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; @@ -35,7 +36,7 @@ import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState; import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.Sort; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +59,8 @@ public class FacetHelper { return new NodeStateFacetsConfig(definition); } - public static Facets getFacets(IndexSearcher searcher, Query query, TopDocs docs, QueryIndex.IndexPlan plan, boolean secure) throws IOException { + public static Facets getFacets(IndexSearcher searcher, Query query, QueryIndex.IndexPlan plan, + SecureFacetConfiguration secureFacetConfiguration) throws IOException { Facets facets = null; @SuppressWarnings("unchecked") List facetFields = (List) plan.getAttribute(ATTR_FACET_FIELDS); @@ -70,10 +72,23 @@ public class FacetHelper { try { DefaultSortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState( searcher.getIndexReader(), FieldNames.createFacetFieldName(facetField)); - FacetsCollector.search(searcher, query, 10, facetsCollector); - facetsMap.put(facetField, secure ? - new FilteredSortedSetDocValuesFacetCounts(state, facetsCollector, plan.getFilter(), docs) : - new SortedSetDocValuesFacetCounts(state, facetsCollector)); + FacetsCollector.search(searcher, query, null,1, Sort.INDEXORDER, facetsCollector); + + switch (secureFacetConfiguration.getMode()) { + case INSECURE: + facets = new SortedSetDocValuesFacetCounts(state, facetsCollector); + break; + case STATISTICAL: + facets = new StatisticalSortedSetDocValuesFacetCounts(state, facetsCollector, plan.getFilter(), + secureFacetConfiguration); + break; + case SECURE: + default: + facets = new SecureSortedSetDocValuesFacetCounts(state, facetsCollector, plan.getFilter()); + break; + } + + facetsMap.put(facetField, facets); } catch (IllegalArgumentException iae) { LOGGER.warn("facets for {} not yet indexed", facetField); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FilteredSortedSetDocValuesFacetCounts.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java similarity index 83% rename from oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FilteredSortedSetDocValuesFacetCounts.java rename to oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java index 5b4ec83693..0a90f0f5e3 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FilteredSortedSetDocValuesFacetCounts.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugins.index.lucene.util; import java.io.IOException; +import java.util.List; import java.util.Map; import com.google.common.collect.Maps; @@ -27,6 +28,7 @@ import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.lucene.document.Document; import org.apache.lucene.facet.FacetResult; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollector.MatchingDocs; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.LabelAndValue; import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState; @@ -35,25 +37,25 @@ import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; import org.jetbrains.annotations.NotNull; /** * ACL filtered version of {@link SortedSetDocValuesFacetCounts} */ -class FilteredSortedSetDocValuesFacetCounts extends SortedSetDocValuesFacetCounts { +class SecureSortedSetDocValuesFacetCounts extends SortedSetDocValuesFacetCounts { - private final TopDocs docs; + private final FacetsCollector facetsCollector; private final Filter filter; private final IndexReader reader; private final SortedSetDocValuesReaderState state; - public FilteredSortedSetDocValuesFacetCounts(DefaultSortedSetDocValuesReaderState state, FacetsCollector facetsCollector, Filter filter, TopDocs docs) throws IOException { + public SecureSortedSetDocValuesFacetCounts(DefaultSortedSetDocValuesReaderState state, FacetsCollector facetsCollector, Filter filter) throws IOException { super(state, facetsCollector); this.reader = state.origReader; + this.facetsCollector = facetsCollector; this.filter = filter; - this.docs = docs; this.state = state; } @@ -68,8 +70,17 @@ class FilteredSortedSetDocValuesFacetCounts extends SortedSetDocValuesFacetCount LabelAndValue[] labelAndValues = topChildren.labelValues; InaccessibleFacetCountManager inaccessibleFacetCountManager = new InaccessibleFacetCountManager(); - for (ScoreDoc scoreDoc : docs.scoreDocs) { - labelAndValues = filterFacet(scoreDoc.doc, dim, labelAndValues, inaccessibleFacetCountManager); + List matchingDocsList = facetsCollector.getMatchingDocs(); + for (MatchingDocs matchingDocs : matchingDocsList) { + DocIdSet bits = matchingDocs.bits; + + DocIdSetIterator docIdSetIterator = bits.iterator(); + int doc = docIdSetIterator.nextDoc(); + while (doc != DocIdSetIterator.NO_MORE_DOCS) { + int docId = matchingDocs.context.docBase + doc; + labelAndValues = filterFacet(docId, dim, labelAndValues, inaccessibleFacetCountManager); + doc = docIdSetIterator.nextDoc(); + } } int childCount = labelAndValues.length; diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java new file mode 100644 index 0000000000..07ebeabe82 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene.util; + +import com.google.common.base.Stopwatch; +import com.google.common.collect.AbstractIterator; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration; +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.lucene.document.Document; +import org.apache.lucene.facet.FacetResult; +import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollector.MatchingDocs; +import org.apache.lucene.facet.LabelAndValue; +import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState; +import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts; +import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.DocIdSetIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +/** + * ACL filtered version of {@link SortedSetDocValuesFacetCounts} + */ +class StatisticalSortedSetDocValuesFacetCounts extends SortedSetDocValuesFacetCounts { + private static final Logger LOG = LoggerFactory.getLogger(StatisticalSortedSetDocValuesFacetCounts.class); + + private final FacetsCollector facetsCollector; + private final Filter filter; + private final IndexReader reader; + private final SortedSetDocValuesReaderState state; + private final SecureFacetConfiguration secureFacetConfiguration; + + public StatisticalSortedSetDocValuesFacetCounts(DefaultSortedSetDocValuesReaderState state, + FacetsCollector facetsCollector, Filter filter, + SecureFacetConfiguration secureFacetConfiguration) throws IOException { + super(state, facetsCollector); + this.reader = state.origReader; + this.facetsCollector = facetsCollector; + this.filter = filter; + this.state = state; + this.secureFacetConfiguration = secureFacetConfiguration; + } + + @Override + public FacetResult getTopChildren(int topN, String dim, String... path) throws IOException { + FacetResult topChildren = super.getTopChildren(topN, dim, path); + + if (topChildren == null) { + return null; + } + + LabelAndValue[] labelAndValues = topChildren.labelValues; + + List matchingDocsList = facetsCollector.getMatchingDocs(); + + int hitCount = 0; + for (MatchingDocs matchingDocs : matchingDocsList) { + hitCount += matchingDocs.totalHits; + } + int sampleSize = secureFacetConfiguration.getStatisticalFacetSampleSize(); + long randomSeed = secureFacetConfiguration.getRandomSeed(); + + LOG.debug("Sampling facet dim {}; hitCount: {}, sampleSize: {}, seed: {}", dim, hitCount, sampleSize, randomSeed); + + Stopwatch w = Stopwatch.createStarted(); + Iterator docIterator = getMatchingDocIterator(matchingDocsList); + Iterator sampleIterator = docIterator; + if (sampleSize < hitCount) { + sampleIterator = getSampledMatchingDocIterator(docIterator, randomSeed, hitCount, sampleSize); + } else { + sampleSize = hitCount; + } + int accessibleSampleCount = getAccessibleSampleCount(dim, sampleIterator); + w.stop(); + + LOG.debug("Evaluated accessible samples {} in {}", accessibleSampleCount, w); + + labelAndValues = updateLabelAndValueIfRequired(labelAndValues, sampleSize, accessibleSampleCount); + + int childCount = labelAndValues.length; + Number value = 0; + for (LabelAndValue lv : labelAndValues) { + value = value.longValue() + lv.value.longValue(); + } + + return new FacetResult(dim, path, value, labelAndValues, childCount); + } + + private Iterator getMatchingDocIterator(final List matchingDocsList) { + Iterator matchingDocsListIterator = matchingDocsList.iterator(); + Iterator matchingDocsIter = new AbstractIterator() { + MatchingDocs matchingDocs = null; + DocIdSetIterator docIdSetIterator = null; + int nextDocId = NO_MORE_DOCS; + @Override + protected Integer computeNext() { + try { + loadNextMatchingDocsIfRequired(); + + if (nextDocId == NO_MORE_DOCS) { + return endOfData(); + } else { + int ret = nextDocId; + nextDocId = docIdSetIterator.nextDoc(); + return ret; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void loadNextMatchingDocsIfRequired() throws IOException { + while (nextDocId == NO_MORE_DOCS) { + if (matchingDocsListIterator.hasNext()) { + matchingDocs = matchingDocsListIterator.next(); + docIdSetIterator = matchingDocs.bits.iterator(); + nextDocId = docIdSetIterator.nextDoc(); + } else { + return; + } + } + } + }; + + return matchingDocsIter; + } + + private Iterator getSampledMatchingDocIterator(Iterator matchingDocs, + long randomdSeed, int hitCount, int sampleSize) { + TapeSampling tapeSampling = new TapeSampling<>(new Random(randomdSeed), matchingDocs, hitCount, sampleSize); + + return tapeSampling.getSamples(); + } + + private int getAccessibleSampleCount(String dim, Iterator sampleIterator) throws IOException { + int count = 0; + while (sampleIterator.hasNext()) { + int docId = sampleIterator.next(); + Document doc = reader.document(docId); + + if (filter.isAccessible(doc.getField(FieldNames.PATH).stringValue() + "/" + dim)) { + count++; + } + } + + return count; + } + + private LabelAndValue[] updateLabelAndValueIfRequired(LabelAndValue[] labelAndValues, + int sampleSize, int accessibleCount) { + if (accessibleCount < sampleSize) { + LabelAndValue[] ret = new LabelAndValue[labelAndValues.length]; + + + for (int i = 0; i < labelAndValues.length; i++) { + LabelAndValue lv = labelAndValues[i]; + ret[i] = new LabelAndValue(lv.label, Math.floorDiv(lv.value.longValue() * accessibleCount, sampleSize)); + } + + return ret; + } else { + return labelAndValues; + } + } +} \ No newline at end of file diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java index 7a1ede4370..3039dafdef 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java @@ -133,14 +133,7 @@ import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.newNodeAgg import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.useV2; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorTest.createCal; import static org.apache.jackrabbit.oak.plugins.index.property.OrderedIndex.OrderDirection; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.ORDERED_PROP_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_ANALYZED; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NAME; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE_SCOPE_INDEX; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_PROPERTY_INDEX; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_TYPE; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.*; import static org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE; import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; import static org.apache.jackrabbit.oak.spi.filter.PathFilter.PROP_EXCLUDED_PATHS; @@ -2908,6 +2901,7 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { idxb.indexRule("nt:base").property("foo").propertyIndex(); Tree idx = root.getTree("/").getChild("oak:index").addChild("test1"); idxb.build(idx); + idx.setProperty(PROP_RANDOM_SEED, new Random().nextLong()); root.commit(); AsyncIndexInfoService asyncService = new AsyncIndexInfoServiceImpl(nodeStore); @@ -3153,6 +3147,124 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { } } + @Test + public void injectRandomSeedDuringReindex() throws Exception{ + IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync(); + idxb.indexRule("nt:base").property("foo").propertyIndex(); + Tree idx = root.getTree("/").getChild("oak:index").addChild("test1"); + idxb.build(idx); + + root.commit(); + + // Push a change to get another indexing cycle run + root.getTree("/").addChild("force-index-run"); + root.commit(); + + NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(), idx.getPath()); + + long defSeed = idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + long clonedSeed = idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + + assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed + " seeds aren't same", defSeed == clonedSeed); + } + + @Test + public void injectRandomSeedDuringRefresh() throws Exception{ + IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync(); + idxb.indexRule("nt:base").property("foo").propertyIndex(); + Tree idx = root.getTree("/").getChild("oak:index").addChild("test1"); + idxb.build(idx); + + root.commit(); + + long seed = new Random().nextLong(); + + // Push a change to get another indexing cycle run + idx.setProperty(PROP_RANDOM_SEED, seed); + idx.setProperty(PROP_REFRESH_DEFN, true); + root.commit(); + + NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(), idx.getPath()); + + long defSeed = idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + long clonedSeed = idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + + assertEquals("Random seed not updated", seed, defSeed); + assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed + " seeds aren't same", defSeed == clonedSeed); + } + + @Test + public void injectRandomSeedDuringUpdate() throws Exception{ + IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync(); + idxb.indexRule("nt:base").property("foo").propertyIndex(); + Tree idx = root.getTree("/").getChild("oak:index").addChild("test1"); + idxb.build(idx); + + root.commit(); + + long seed = new Random().nextLong(); + + // Update seed + idx.setProperty(PROP_RANDOM_SEED, seed); + root.commit(); + + NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(), idx.getPath()); + + long defSeed = idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + long clonedSeed = idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + + assertEquals("Random seed not updated", seed, defSeed); + assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed + " seeds aren't same", defSeed == clonedSeed); + } + + @Test + public void injectGarbageSeed() throws Exception { + IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync(); + idxb.indexRule("nt:base").property("foo").propertyIndex(); + Tree idx = root.getTree("/").getChild("oak:index").addChild("test1"); + idxb.build(idx); + + root.commit(); + + long orignalSeed = NodeStateUtils.getNode(nodeStore.getRoot(), idx.getPath()) + .getProperty(PROP_RANDOM_SEED) + .getValue(Type.LONG); + + // Update seed + idx.setProperty(PROP_RANDOM_SEED, "garbage"); + root.commit(); + + NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(), idx.getPath()); + + long defSeed = idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + long clonedSeed = idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + + assertNotEquals("Random seed not updated", orignalSeed, defSeed); + assertNotEquals("Random seed updated to garbage", "garbage", defSeed); + assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed + " seeds aren't same", defSeed == clonedSeed); + } + + @Test + public void injectStringLongSeed() throws Exception { + IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync(); + idxb.indexRule("nt:base").property("foo").propertyIndex(); + Tree idx = root.getTree("/").getChild("oak:index").addChild("test1"); + idxb.build(idx); + + root.commit(); + // Update seed + idx.setProperty(PROP_RANDOM_SEED, "-12312"); + root.commit(); + + NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(), idx.getPath()); + + long defSeed = idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + long clonedSeed = idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG); + + assertEquals("Random seed udpated to garbage", -12312, defSeed); + assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed + " seeds aren't same", defSeed == clonedSeed); + } + private void assertPlanAndQuery(String query, String planExpectation, List paths) { assertPlanAndQuery(query, planExpectation, paths, false); } diff --git a/oak-search/pom.xml b/oak-search/pom.xml index ef1038a976..1ed740de73 100644 --- a/oak-search/pom.xml +++ b/oak-search/pom.xml @@ -162,6 +162,13 @@ tests test + + org.apache.jackrabbit + oak-commons + ${project.version} + test-jar + test + diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java index b13c672af4..c52b9e554f 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java @@ -306,6 +306,15 @@ public interface FulltextIndexConstants { */ String PROP_SECURE_FACETS = "secure"; + String PROP_SECURE_FACETS_VALUE_INSECURE = "insecure"; + String PROP_SECURE_FACETS_VALUE_STATISTICAL = "statistical"; + String PROP_SECURE_FACETS_VALUE_SECURE = "secure"; + String PROP_SECURE_FACETS_VALUE_JVM_PARAM = "oak.facets.secure"; + + String STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM = "oak.facet.statistical.sampleSize"; + String PROP_STATISTICAL_FACET_SAMPLE_SIZE = "sampleSize"; + int STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT = 1000; + /** * Optional (index definition) property indicating max number of facets that will be retrieved * in query @@ -334,6 +343,13 @@ public interface FulltextIndexConstants { */ String PROP_REFRESH_DEFN = "refresh"; + /** + * Long property that keep seed for random number generation. One example usage of this is + * to randomly sample query results to statistically check for ACLs to extrapolate facet + * counts + */ + String PROP_RANDOM_SEED = "seed"; + /** * Boolean property to indicate that nodes nodetype matching indexRule name * should be indexed diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java index 261134cf33..ee03dae49a 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.regex.Pattern; import javax.jcr.PropertyType; @@ -79,25 +80,7 @@ import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.ENTRY_COUNT import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEXING_MODE_NRT; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEXING_MODE_SYNC; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.BLOB_SIZE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.COMPAT_MODE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EVALUATE_PATH_RESTRICTION; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EXCLUDE_PROPERTY_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EXPERIMENTAL_STORAGE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FACETS; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FIELD_BOOST; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FULL_TEXT_ENABLED; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_TYPES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_DATA_CHILD_NAME; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.ORDERED_PROP_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_FACETS_TOP_CHILDREN; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_CONFIG; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_MAPPED_TYPE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_MIME_TYPES; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.*; import static org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition.DEFAULT_BOOST; import static org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil.getOptionalValue; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; @@ -244,7 +227,8 @@ public class IndexDefinition implements Aggregate.AggregateMapper { private final boolean suggestAnalyzed; - private final boolean secureFacets; + private final SecureFacetConfiguration secureFacets; + private final long randomSeed; private final int numberOfTopFacets; @@ -411,12 +395,21 @@ public class IndexDefinition implements Aggregate.AggregateMapper { this.queryPaths = getOptionalStrings(defn, IndexConstants.QUERY_PATHS); this.suggestAnalyzed = evaluateSuggestAnalyzed(defn, false); + { + PropertyState randomPS = defn.getProperty(PROP_RANDOM_SEED); + if (randomPS != null && randomPS.getType() == Type.LONG) { + randomSeed = randomPS.getValue(Type.LONG); + } else { + // create a random number + randomSeed = UUID.randomUUID().getMostSignificantBits(); + } + } if (defn.hasChildNode(FACETS)) { NodeState facetsConfig = defn.getChildNode(FACETS); - this.secureFacets = getOptionalValue(facetsConfig, PROP_SECURE_FACETS, true); + this.secureFacets = SecureFacetConfiguration.getInstance(randomSeed, facetsConfig); this.numberOfTopFacets = getOptionalValue(facetsConfig, PROP_FACETS_TOP_CHILDREN, DEFAULT_FACET_COUNT); } else { - this.secureFacets = true; + this.secureFacets = SecureFacetConfiguration.getInstance(randomSeed, null); this.numberOfTopFacets = DEFAULT_FACET_COUNT; } @@ -879,7 +872,7 @@ public class IndexDefinition implements Aggregate.AggregateMapper { return suggestAnalyzed; } - public boolean isSecureFacets() { + public SecureFacetConfiguration getSecureFacetConfiguration() { return secureFacets; } @@ -1814,4 +1807,97 @@ public class IndexDefinition implements Aggregate.AggregateMapper { return new PropertyDefinition(rule, name, builder.getNodeState()); } + public static class SecureFacetConfiguration { + public enum MODE { + SECURE, + STATISTICAL, + INSECURE + } + + private final long randomSeed; + private final MODE mode; + private final int statisticalFacetSampleSize; + + SecureFacetConfiguration(long randomSeed, MODE mode, int statisticalFacetSampleSize) { + this.randomSeed = randomSeed; + this.mode = mode; + this.statisticalFacetSampleSize = statisticalFacetSampleSize; + } + + public MODE getMode() { + return mode; + } + + public int getStatisticalFacetSampleSize() { + return statisticalFacetSampleSize; + } + + public long getRandomSeed() { + return randomSeed; + } + + static SecureFacetConfiguration getInstance(long randomSeed, NodeState facetConfigRoot) { + if (facetConfigRoot == null) { + facetConfigRoot = EMPTY_NODE; + } + + MODE mode = getMode(facetConfigRoot); + int statisticalFacetSampleSize; + + if (mode == MODE.STATISTICAL) { + statisticalFacetSampleSize = getStatisticalFacetSampleSize(facetConfigRoot); + } else { + statisticalFacetSampleSize = -1; + } + + return new SecureFacetConfiguration(randomSeed, mode, statisticalFacetSampleSize); + } + + static MODE getMode(@NotNull final NodeState facetConfigRoot) { + PropertyState securePS = facetConfigRoot.getProperty(PROP_SECURE_FACETS); + String modeString; + + if (securePS != null) { + if(securePS.getType() == Type.BOOLEAN) { + // legacy secure config + boolean secure = securePS.getValue(Type.BOOLEAN); + return secure ? MODE.SECURE : MODE.INSECURE; + } else { + modeString = securePS.getValue(Type.STRING); + } + } else { + // use "default" just as placeholder. default case would do the actual default eval + modeString = System.getProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "default"); + } + + switch (modeString) { + case PROP_SECURE_FACETS_VALUE_INSECURE: + return MODE.INSECURE; + case PROP_SECURE_FACETS_VALUE_STATISTICAL: + return MODE.STATISTICAL; + case PROP_SECURE_FACETS_VALUE_SECURE: + default: + return MODE.SECURE; + } + } + + static int getStatisticalFacetSampleSize(@NotNull final NodeState facetConfigRoot) { + int statisticalFacetSampleSize = Integer.getInteger( + STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, + STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT); + int statisticalFacetSampleSizePV = getOptionalValue(facetConfigRoot, + PROP_STATISTICAL_FACET_SAMPLE_SIZE, + statisticalFacetSampleSize); + + if (statisticalFacetSampleSizePV > 0) { + statisticalFacetSampleSize = statisticalFacetSampleSizePV; + } + + if (statisticalFacetSampleSize < 0) { + statisticalFacetSampleSize = STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT; + } + + return statisticalFacetSampleSize; + } + } } diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java index 48d0948c9a..a20a55d111 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java @@ -20,8 +20,10 @@ package org.apache.jackrabbit.oak.plugins.index.search.spi.editor; import java.io.IOException; import java.util.Calendar; +import java.util.UUID; import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PerfLogger; import org.apache.jackrabbit.oak.plugins.index.IndexConstants; @@ -44,6 +46,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_RANDOM_SEED; import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_REFRESH_DEFN; import static org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE; @@ -248,6 +251,16 @@ public abstract class FulltextIndexEditorContext { private IndexDefinition createIndexDefinition(NodeState root, NodeBuilder definition, IndexingContext indexingContext, boolean asyncIndexing) { + + // A good time to check and see if we want to inject our random + // seed - required due to OAK-7929 but could be useful otherwise too. + Long defRandom = getLongPropertyOrNull(definition.getProperty(PROP_RANDOM_SEED)); + if (defRandom == null) { + long seed = UUID.randomUUID().getMostSignificantBits(); + definition.setProperty(PROP_RANDOM_SEED, seed); + defRandom = seed; + } + NodeState defnState = definition.getBaseState(); if (asyncIndexing && !IndexDefinition.isDisableStoredIndexDefinition()){ if (definition.getBoolean(PROP_REFRESH_DEFN)){ @@ -262,6 +275,17 @@ public abstract class FulltextIndexEditorContext { definition.setChildNode(INDEX_DEFINITION_NODE, NodeStateCloner.cloneVisibleState(defnState)); log.info("Stored the cloned index definition for [{}]. Changes in index definition would now only be " + "effective post reindexing", indexingContext.getIndexPath()); + } else { + // This is neither reindex nor refresh. So, let's update cloned def with random seed + // if it doesn't match what's there in main definition + // Reindex would require another indexing cycle to update cloned def. + // Refresh would anyway is going to do a real clone so that's ok + NodeState clonedState = defnState.getChildNode(INDEX_DEFINITION_NODE); + Long clonedRandom = getLongPropertyOrNull(clonedState.getProperty(PROP_RANDOM_SEED)); + + if (clonedRandom == null || clonedRandom != defRandom) { + definition.getChildNode(INDEX_DEFINITION_NODE).setProperty(PROP_RANDOM_SEED, defRandom); + } } } return newDefinitionBuilder() @@ -270,4 +294,18 @@ public abstract class FulltextIndexEditorContext { .indexPath(indexingContext.getIndexPath()) .build(); } + + private Long getLongPropertyOrNull(PropertyState propertyState) { + Long ret = null; + + if (propertyState != null) { + try { + ret = propertyState.getValue(Type.LONG); + } catch (Exception e) { + // ignored + } + } + + return ret; + } } diff --git a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java new file mode 100644 index 0000000000..bd643e5f75 --- /dev/null +++ b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.search; + +import org.apache.jackrabbit.oak.commons.junit.TemporarySystemProperty; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Random; + +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.*; +import static org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration.MODE; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +public class IndexDefinitionFacetConfigTest { + + @Rule + public TemporarySystemProperty temporarySystemProperty = new TemporarySystemProperty(); + + private NodeState root = EMPTY_NODE; + + private final NodeBuilder builder = root.builder(); + + private static final long RANDOM_SEED = 1L; + + @Test + public void defaultConfig() { + SecureFacetConfiguration config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root); + + assertEquals(config.getMode(), MODE.SECURE); + + int sampleSize = config.getStatisticalFacetSampleSize(); + assertEquals(-1, sampleSize); + } + + @Test + public void nonSecureConfigMode() { + SecureFacetConfiguration config; + + builder.setProperty(PROP_SECURE_FACETS, "insecure"); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, builder.getNodeState()); + assertEquals(-1, config.getStatisticalFacetSampleSize()); + + builder.setProperty(PROP_SECURE_FACETS, "statistical"); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, builder.getNodeState()); + assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, config.getStatisticalFacetSampleSize()); + } + + @Test + public void systemPropSecureFacet() { + SecureFacetConfiguration config; + + System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "random"); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root); + assertEquals(MODE.SECURE, config.getMode()); + + System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "secure"); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root); + assertEquals(MODE.SECURE, config.getMode()); + + System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "insecure"); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root); + assertEquals(MODE.INSECURE, config.getMode()); + + System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "insecure"); + builder.setProperty(PROP_SECURE_FACETS, "secure"); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, builder.getNodeState()); + assertEquals(MODE.SECURE, config.getMode()); + } + + @Test + public void systemPropSecureFacetStatisticalSampleSize() { + int sampleSize; + + System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, PROP_SECURE_FACETS_VALUE_STATISTICAL); + + System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "10"); + sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED, root).getStatisticalFacetSampleSize(); + assertEquals(10, sampleSize); + + System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "-10"); + sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED, root).getStatisticalFacetSampleSize(); + assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize); + + System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "100000000000"); + sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED, root).getStatisticalFacetSampleSize(); + assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize); + } + + @Test + public void invalidSecureFacetSampleSize() { + int sampleSize; + NodeBuilder configBuilder = builder.child("config").setProperty(PROP_SECURE_FACETS, PROP_SECURE_FACETS_VALUE_STATISTICAL); + NodeState nodeState = configBuilder.getNodeState(); + + configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, -10); + sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState).getStatisticalFacetSampleSize(); + assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize); + + System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "10"); + configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, -20); + nodeState = configBuilder.getNodeState(); + sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState).getStatisticalFacetSampleSize(); + assertEquals(10, sampleSize); + + System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "-10"); + configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, -20); + sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState).getStatisticalFacetSampleSize(); + assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize); + + System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "-10"); + configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, 10); + nodeState = configBuilder.getNodeState(); + sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState).getStatisticalFacetSampleSize(); + assertEquals(10, sampleSize); + } + + @Test + public void orderingOfOverrides() { + System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "insecure"); + System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "10"); + + NodeState nodeState; + SecureFacetConfiguration config; + int sampleSize; + + nodeState = builder.child("config1").setProperty(PROP_SECURE_FACETS, "secure").getNodeState(); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState); + assertEquals(MODE.SECURE, config.getMode()); + assertEquals(-1, config.getStatisticalFacetSampleSize()); + + nodeState = builder.child("config2") + .setProperty(PROP_SECURE_FACETS, PROP_SECURE_FACETS_VALUE_STATISTICAL) + .setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, 20) + .getNodeState(); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState); + sampleSize = config.getStatisticalFacetSampleSize(); + assertEquals(20, sampleSize); + + nodeState = builder.child("config3") + .setProperty(PROP_SECURE_FACETS, PROP_SECURE_FACETS_VALUE_STATISTICAL) + .getNodeState(); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState); + sampleSize = config.getStatisticalFacetSampleSize(); + assertEquals(10, sampleSize); + } + + @Test + public void legacyConfig() { + NodeState ns = builder.setProperty(PROP_SECURE_FACETS, true).getNodeState(); + SecureFacetConfiguration config = SecureFacetConfiguration.getInstance(RANDOM_SEED, ns); + assertEquals(MODE.SECURE, config.getMode()); + assertEquals(-1, config.getStatisticalFacetSampleSize()); + + ns = builder.setProperty(PROP_SECURE_FACETS, false).getNodeState(); + config = SecureFacetConfiguration.getInstance(RANDOM_SEED, ns); + assertEquals(MODE.INSECURE, config.getMode()); + assertEquals(-1, config.getStatisticalFacetSampleSize()); + } + + @Test + public void absentFacetConfigNode() { + IndexDefinition idxDefn = new IndexDefinition(root, root, "/foo"); + + SecureFacetConfiguration config = idxDefn.getSecureFacetConfiguration(); + + assertEquals(MODE.SECURE, config.getMode()); + } + + @Test + public void randomSeed() { + long seed = new Random().nextLong(); + builder.setProperty(PROP_RANDOM_SEED, seed); + root = builder.getNodeState(); + IndexDefinition idxDefn = new IndexDefinition(root, root, "/foo"); + + SecureFacetConfiguration config = idxDefn.getSecureFacetConfiguration(); + + assertEquals(seed, config.getRandomSeed()); + } + + @Test + public void randomSeedWithoutOneInDef() { + long seed1 = new IndexDefinition(root, root, "/foo").getSecureFacetConfiguration().getRandomSeed(); + long seed2 = new IndexDefinition(root, root, "/foo").getSecureFacetConfiguration().getRandomSeed(); + + assertNotEquals(seed1, seed2); + } +} \ No newline at end of file -- 2.18.0