Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/AmortizedSampleFixer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/AmortizedSampleFixer.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/AmortizedSampleFixer.java (working copy) @@ -0,0 +1,42 @@ +package org.apache.lucene.facet.sampling; + +import java.io.IOException; + +import org.apache.lucene.facet.search.FacetResultNode; +import org.apache.lucene.facet.search.ScoredDocIDs; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Fix sampling results by assuming the sample represents the matched docs + * exactly, dividing the values by the sampled ratio. E.g if the sampled set + * was 5% of the matched documents, the values would be multiplied by 20. + *

+ * This fixer is suitable for scenarios which prioritize performance over + * accuracy. For scenarios prioritizing accuracy over performance, see + * {@link TakmiSampleFixer} + * + * @lucene.experimental + */ +public class AmortizedSampleFixer extends SampleFixer { + @Override + public void singleNodeFix(FacetResultNode facetResNode, ScoredDocIDs docIds, double samplingRatio) + throws IOException { + facetResNode.value /= samplingRatio; + } +} Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/SampleFixer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/SampleFixer.java (revision 1485933) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/SampleFixer.java (working copy) @@ -3,6 +3,7 @@ import java.io.IOException; import org.apache.lucene.facet.search.FacetResult; +import org.apache.lucene.facet.search.FacetResultNode; import org.apache.lucene.facet.search.ScoredDocIDs; /* @@ -27,7 +28,7 @@ * * @lucene.experimental */ -public interface SampleFixer { +public abstract class SampleFixer { /** * Alter the input result, fixing it to account for the sampling. This @@ -40,5 +41,26 @@ * sample result to be fixed. * @throws IOException If there is a low-level I/O error. */ - public void fixResult(ScoredDocIDs origDocIds, FacetResult fres) throws IOException; + public void fixResult(ScoredDocIDs origDocIds, FacetResult fres, double samplingRatio) throws IOException { + FacetResultNode topRes = fres.getFacetResultNode(); + fixResultNode(topRes, origDocIds ,samplingRatio); + }; + + /** + * Fix result node count, and, recursively, fix all its children + * + * @param facetResNode + * result node to be fixed + * @param docIds + * docids in effect + * @throws IOException If there is a low-level I/O error. + */ + protected void fixResultNode(FacetResultNode facetResNode, ScoredDocIDs docIds, double samplingRatio) throws IOException { + singleNodeFix(facetResNode, docIds, samplingRatio); + for (FacetResultNode frn : facetResNode.subResults) { + fixResultNode(frn, docIds, samplingRatio); + } + } + + protected void singleNodeFix(FacetResultNode facetResNode, ScoredDocIDs docIds, double samplingRatio) throws IOException {} } \ No newline at end of file Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/Sampler.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/Sampler.java (revision 1485933) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/Sampler.java (working copy) @@ -117,7 +117,11 @@ */ public SampleFixer getSampleFixer(IndexReader indexReader, TaxonomyReader taxonomyReader, FacetSearchParams searchParams) { - return new TakmiSampleFixer(indexReader, taxonomyReader, searchParams); + if (samplingParams.shouldFixToExact()) { + return new TakmiSampleFixer(indexReader, taxonomyReader, searchParams); + } else { + return new AmortizedSampleFixer(); + } } /** Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingAccumulator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingAccumulator.java (revision 1485933) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingAccumulator.java (working copy) @@ -79,7 +79,11 @@ public List accumulate(ScoredDocIDs docids) throws IOException { // Replacing the original searchParams with the over-sampled FacetSearchParams original = searchParams; - searchParams = sampler.overSampledSearchParams(original); + SampleFixer samplerFixer = sampler.getSampleFixer(indexReader, taxonomyReader, original); + final boolean fixToExact = sampler.samplingParams.shouldFixToExact(); + if (fixToExact) { + searchParams = sampler.overSampledSearchParams(original); + } List sampleRes = super.accumulate(docids); @@ -88,13 +92,15 @@ // for sure fres is not null because this is guaranteed by the delegee. PartitionsFacetResultsHandler frh = createFacetResultsHandler(fres.getFacetRequest()); // fix the result of current request - sampler.getSampleFixer(indexReader, taxonomyReader, searchParams).fixResult(docids, fres); + samplerFixer.fixResult(docids, fres, samplingRatio); fres = frh.rearrangeFacetResult(fres); // let delegee's handler do any arranging it needs to - // Using the sampler to trim the extra (over-sampled) results - fres = sampler.trimResult(fres); - + if (fixToExact) { + // Using the sampler to trim the extra (over-sampled) results + fres = sampler.trimResult(fres); + } + // final labeling if allowed (because labeling is a costly operation) frh.labelResult(fres); fixedRes.add(fres); // add to final results Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingParams.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingParams.java (revision 1485933) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingParams.java (working copy) @@ -54,11 +54,19 @@ */ public static final int DEFAULT_SAMPLING_THRESHOLD = 75000; + /** + * Default fix-to-exact to false, meaning the default would produce amortized + * results (faster) than exact results (slower) + */ + public static final boolean DEFAULT_FIX_TO_EXACT = false; + private int maxSampleSize = DEFAULT_MAX_SAMPLE_SIZE; private int minSampleSize = DEFAULT_MIN_SAMPLE_SIZE; private double sampleRatio = DEFAULT_SAMPLE_RATIO; private int samplingThreshold = DEFAULT_SAMPLING_THRESHOLD; private double oversampleFactor = DEFAULT_OVERSAMPLE_FACTOR; + + private boolean fixToExact = DEFAULT_FIX_TO_EXACT; /** * Return the maxSampleSize. @@ -166,4 +174,14 @@ this.oversampleFactor = oversampleFactor; } + /** Donates whether the fixing would result with true, exact, values as if no sampling had occur */ + public boolean shouldFixToExact() { + return fixToExact; + } + + /** @see #shouldFixToExact() */ + public void setFixToExact(boolean fixToExact) { + this.fixToExact = fixToExact; + } + } \ No newline at end of file Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingWrapper.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingWrapper.java (revision 1485933) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingWrapper.java (working copy) @@ -63,7 +63,7 @@ // for sure fres is not null because this is guaranteed by the delegee. PartitionsFacetResultsHandler frh = createFacetResultsHandler(fres.getFacetRequest()); // fix the result of current request - sampler.getSampleFixer(indexReader, taxonomyReader, searchParams).fixResult(docids, fres); + sampler.getSampleFixer(indexReader, taxonomyReader, searchParams).fixResult(docids, fres, sampleSet.actualSampleRatio); fres = frh.rearrangeFacetResult(fres); // let delegee's handler do any // Using the sampler to trim the extra (over-sampled) results Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/TakmiSampleFixer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/TakmiSampleFixer.java (revision 1485933) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/TakmiSampleFixer.java (working copy) @@ -2,21 +2,19 @@ import java.io.IOException; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.Bits; - import org.apache.lucene.facet.params.FacetSearchParams; import org.apache.lucene.facet.search.DrillDownQuery; -import org.apache.lucene.facet.search.FacetResult; import org.apache.lucene.facet.search.FacetResultNode; import org.apache.lucene.facet.search.ScoredDocIDs; import org.apache.lucene.facet.search.ScoredDocIDsIterator; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.Bits; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -36,16 +34,18 @@ */ /** - * Fix sampling results by counting the intersection between two lists: a - * TermDocs (list of documents in a certain category) and a DocIdSetIterator - * (list of documents matching the query). + * Fix sampling results by correct results, by counting the intersection between + * two lists: a TermDocs (list of documents in a certain category) and a + * DocIdSetIterator (list of documents matching the query). + *

+ * This fixer is suitable for scenarios which prioritize accuracy over + * performance. For scenarios prioritizing performance over accuracy, see + * {@link AmortizedSampleFixer} * * * @lucene.experimental */ -// TODO (Facet): implement also an estimated fixing by ratio (taking into -// account "translation" of counts!) -class TakmiSampleFixer implements SampleFixer { +class TakmiSampleFixer extends SampleFixer { private TaxonomyReader taxonomyReader; private IndexReader indexReader; @@ -59,29 +59,11 @@ } @Override - public void fixResult(ScoredDocIDs origDocIds, FacetResult fres) - throws IOException { - FacetResultNode topRes = fres.getFacetResultNode(); - fixResultNode(topRes, origDocIds); + public void singleNodeFix(FacetResultNode facetResNode, ScoredDocIDs docIds, double samplingRatio) throws IOException { + recount(facetResNode, docIds); } /** - * Fix result node count, and, recursively, fix all its children - * - * @param facetResNode - * result node to be fixed - * @param docIds - * docids in effect - * @throws IOException If there is a low-level I/O error. - */ - private void fixResultNode(FacetResultNode facetResNode, ScoredDocIDs docIds) throws IOException { - recount(facetResNode, docIds); - for (FacetResultNode frn : facetResNode.subResults) { - fixResultNode(frn, docIds); - } - } - - /** * Internal utility: recount for a facet result node * * @param fresNode @@ -179,4 +161,5 @@ } return false; // exhausted } + } \ No newline at end of file Index: lucene/facet/src/test/org/apache/lucene/facet/sampling/AmortizedSamplerFixerTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/sampling/AmortizedSamplerFixerTest.java (revision 0) +++ lucene/facet/src/test/org/apache/lucene/facet/sampling/AmortizedSamplerFixerTest.java (working copy) @@ -0,0 +1,55 @@ +package org.apache.lucene.facet.sampling; + +import java.util.ArrayList; + +import org.apache.lucene.facet.FacetTestBase; +import org.apache.lucene.facet.search.CountFacetRequest; +import org.apache.lucene.facet.search.FacetResult; +import org.apache.lucene.facet.search.FacetResultNode; +import org.apache.lucene.facet.taxonomy.CategoryPath; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class AmortizedSamplerFixerTest extends FacetTestBase { + + public void testFixing() throws Exception { + // a random sample ratio at the range of [0.1, 0.6) + final double sampleRatio = 0.1d + (random().nextDouble() / 2d); + + AmortizedSampleFixer asf = new AmortizedSampleFixer(); + FacetResultNode root = new FacetResultNode(1, 11.0d); + final FacetResultNode child1 = new FacetResultNode(2, 7.0d); + final FacetResultNode child2 = new FacetResultNode(3, 4.0d); + final FacetResultNode child3 = new FacetResultNode(4, 3.0d); + root.subResults = new ArrayList() {{ + add(child1); + add(child2); + add(child3); + }}; + + FacetResult res = new FacetResult(new CountFacetRequest(new CategoryPath("path"), 10), root, 10); + asf.fixResult(null, res, sampleRatio); + + assertEquals(11.0 / sampleRatio, root.value, Double.MIN_NORMAL); + assertEquals(7.0 / sampleRatio, child1.value, Double.MIN_NORMAL); + assertEquals(4.0 / sampleRatio, child2.value, Double.MIN_NORMAL); + assertEquals(3.0 / sampleRatio, child3.value, Double.MIN_NORMAL); + } + + +} Index: lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java (revision 1485933) +++ lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java (working copy) @@ -130,6 +130,7 @@ private Sampler createSampler(int nTrial, boolean useRandomSampler) { SamplingParams samplingParams = new SamplingParams(); + samplingParams.setFixToExact(true); // Extremely hard to test amortized results.. final double retryFactor = Math.pow(1.01, nTrial); samplingParams.setSampleRatio(0.8 * retryFactor); Index: lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplerTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplerTest.java (revision 0) +++ lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplerTest.java (working copy) @@ -0,0 +1,47 @@ +package org.apache.lucene.facet.sampling; + +import org.apache.lucene.facet.FacetTestBase; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class SamplerTest extends FacetTestBase { + + public void testDefaultFixer() throws Exception { + RandomSampler randomSampler = new RandomSampler(); + SampleFixer fixer = randomSampler.getSampleFixer(null, null, null); + assertEquals(AmortizedSampleFixer.class, fixer.getClass()); + } + + public void testExactFixer() throws Exception { + SamplingParams sp = new SamplingParams(); + sp.setFixToExact(true); + RandomSampler randomSampler = new RandomSampler(sp, random()); + SampleFixer fixer = randomSampler.getSampleFixer(null, null, null); + assertEquals(TakmiSampleFixer.class, fixer.getClass()); + } + + public void testAmortizedFixer() throws Exception { + SamplingParams sp = new SamplingParams(); + sp.setFixToExact(false); + RandomSampler randomSampler = new RandomSampler(sp, random()); + SampleFixer fixer = randomSampler.getSampleFixer(null, null, null); + assertEquals(AmortizedSampleFixer.class, fixer.getClass()); + } + + +}