Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/AmortizedSampleFixer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/AmortizedSampleFixer.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/AmortizedSampleFixer.java (working copy) @@ -0,0 +1,42 @@ +package org.apache.lucene.facet.sampling; + +import java.io.IOException; + +import org.apache.lucene.facet.search.FacetResultNode; +import org.apache.lucene.facet.search.ScoredDocIDs; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Fix sampling results by assuming the sample represents the matched docs + * exactly, dividing the values by the sampled ratio. E.g if the sampled set + * was 5% of the matched documents, the values would be multiplied by 20. + *

+ * This fixer is suitable for scenarios which prioritize performance over + * accuracy. For scenarios prioritizing accuracy over performance, see + * {@link TakmiSampleFixer} + * + * @lucene.experimental + */ +public class AmortizedSampleFixer extends SampleFixer { + @Override + public void singleNodeFix(FacetResultNode facetResNode, ScoredDocIDs docIds, double samplingRatio) + throws IOException { + facetResNode.value /= samplingRatio; + } +} Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/SampleFixer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/SampleFixer.java (revision 1486398) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/SampleFixer.java (working copy) @@ -3,6 +3,7 @@ import java.io.IOException; import org.apache.lucene.facet.search.FacetResult; +import org.apache.lucene.facet.search.FacetResultNode; import org.apache.lucene.facet.search.ScoredDocIDs; /* @@ -27,7 +28,7 @@ * * @lucene.experimental */ -public interface SampleFixer { +public abstract class SampleFixer { /** * Alter the input result, fixing it to account for the sampling. This @@ -40,5 +41,51 @@ * sample result to be fixed. * @throws IOException If there is a low-level I/O error. */ - public void fixResult(ScoredDocIDs origDocIds, FacetResult fres) throws IOException; + public void fixResult(ScoredDocIDs origDocIds, FacetResult fres, + double samplingRatio) throws IOException { + FacetResultNode topRes = fres.getFacetResultNode(); + fixResultNode(topRes, origDocIds ,samplingRatio); + }; + + /** + * Fix result node count, and, recursively, fix all its children + * + * @param facetResNode + * result node to be fixed + * @param docIds + * docids in effect + * @throws IOException If there is a low-level I/O error. + */ + protected void fixResultNode(FacetResultNode facetResNode, + ScoredDocIDs docIds, double samplingRatio) throws IOException { + singleNodeFix(facetResNode, docIds, samplingRatio); + for (FacetResultNode frn : facetResNode.subResults) { + fixResultNode(frn, docIds, samplingRatio); + } + } + + /** Fix the given node's value */ + protected void singleNodeFix(FacetResultNode facetResNode, + ScoredDocIDs docIds, double samplingRatio) throws IOException {} + + /** A No-Op fixer, does not alter the results at all.*/ + public static final class NoopSampleFixer extends SampleFixer { + @Override + public void fixResult(ScoredDocIDs origDocIds, FacetResult fres, + double samplingRatio) throws IOException { + return; + } + + @Override + protected void fixResultNode(FacetResultNode facetResNode, + ScoredDocIDs docIds, double samplingRatio) throws IOException { + return; + } + + @Override + protected void singleNodeFix(FacetResultNode facetResNode, + ScoredDocIDs docIds, double samplingRatio) throws IOException { + return; + } + } } \ No newline at end of file Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/Sampler.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/Sampler.java (revision 1486398) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/Sampler.java (working copy) @@ -12,7 +12,6 @@ import org.apache.lucene.facet.search.FacetResultNode; import org.apache.lucene.facet.search.ScoredDocIDs; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -import org.apache.lucene.index.IndexReader; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -111,16 +110,6 @@ throws IOException; /** - * Get a fixer of sample facet accumulation results. Default implementation - * returns a TakmiSampleFixer which is adequate only for - * counting. For any other accumulator, provide a different fixer. - */ - public SampleFixer getSampleFixer(IndexReader indexReader, TaxonomyReader taxonomyReader, - FacetSearchParams searchParams) { - return new TakmiSampleFixer(indexReader, taxonomyReader, searchParams); - } - - /** * Result of sample computation */ public final static class SampleResult { Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingAccumulator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingAccumulator.java (revision 1486398) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingAccumulator.java (working copy) @@ -79,7 +79,11 @@ public List accumulate(ScoredDocIDs docids) throws IOException { // Replacing the original searchParams with the over-sampled FacetSearchParams original = searchParams; - searchParams = sampler.overSampledSearchParams(original); + SampleFixer samplerFixer = sampler.samplingParams.getSampleFixer(); + final boolean shouldOversample = sampler.samplingParams.getOversampleFactor() > 1d; + if (shouldOversample) { + searchParams = sampler.overSampledSearchParams(original); + } List sampleRes = super.accumulate(docids); @@ -88,13 +92,15 @@ // for sure fres is not null because this is guaranteed by the delegee. PartitionsFacetResultsHandler frh = createFacetResultsHandler(fres.getFacetRequest()); // fix the result of current request - sampler.getSampleFixer(indexReader, taxonomyReader, searchParams).fixResult(docids, fres); + samplerFixer.fixResult(docids, fres, samplingRatio); fres = frh.rearrangeFacetResult(fres); // let delegee's handler do any arranging it needs to - // Using the sampler to trim the extra (over-sampled) results - fres = sampler.trimResult(fres); - + if (shouldOversample) { + // Using the sampler to trim the extra (over-sampled) results + fres = sampler.trimResult(fres); + } + // final labeling if allowed (because labeling is a costly operation) frh.labelResult(fres); fixedRes.add(fres); // add to final results Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingParams.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingParams.java (revision 1486398) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingParams.java (working copy) @@ -28,7 +28,7 @@ * Default factor by which more results are requested over the sample set. * @see SamplingParams#getOversampleFactor() */ - public static final double DEFAULT_OVERSAMPLE_FACTOR = 2d; + public static final double DEFAULT_OVERSAMPLE_FACTOR = 1d; /** * Default ratio between size of sample to original size of document set. @@ -54,11 +54,18 @@ */ public static final int DEFAULT_SAMPLING_THRESHOLD = 75000; + /** + * Default {@link SampleFixer} which computed {@link AmortizedSampleFixer amortized} results + */ + private static final SampleFixer DEFAULT_SAMPLE_FIXER = new AmortizedSampleFixer(); + private int maxSampleSize = DEFAULT_MAX_SAMPLE_SIZE; private int minSampleSize = DEFAULT_MIN_SAMPLE_SIZE; private double sampleRatio = DEFAULT_SAMPLE_RATIO; private int samplingThreshold = DEFAULT_SAMPLING_THRESHOLD; private double oversampleFactor = DEFAULT_OVERSAMPLE_FACTOR; + + private SampleFixer sampleFixer = DEFAULT_SAMPLE_FIXER; /** * Return the maxSampleSize. @@ -166,4 +173,17 @@ this.oversampleFactor = oversampleFactor; } + /** @return {@link SampleFixer} to be used while fixing the sampled results */ + public SampleFixer getSampleFixer() { + return sampleFixer; + } + + /** Set a {@link SampleFixer} to be used while fixing the sampled results */ + public void setSampleFixer(SampleFixer sampleFixer) { + this.sampleFixer = sampleFixer; + } + + + + } \ No newline at end of file Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingWrapper.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingWrapper.java (revision 1486398) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/SamplingWrapper.java (working copy) @@ -59,11 +59,12 @@ List sampleRes = delegee.accumulate(sampleSet.docids); List fixedRes = new ArrayList(); + SampleFixer sampleFixer = sampler.samplingParams.getSampleFixer(); for (FacetResult fres : sampleRes) { // for sure fres is not null because this is guaranteed by the delegee. PartitionsFacetResultsHandler frh = createFacetResultsHandler(fres.getFacetRequest()); // fix the result of current request - sampler.getSampleFixer(indexReader, taxonomyReader, searchParams).fixResult(docids, fres); + sampleFixer.fixResult(docids, fres, sampleSet.actualSampleRatio); fres = frh.rearrangeFacetResult(fres); // let delegee's handler do any // Using the sampler to trim the extra (over-sampled) results Index: lucene/facet/src/java/org/apache/lucene/facet/sampling/TakmiSampleFixer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sampling/TakmiSampleFixer.java (revision 1486398) +++ lucene/facet/src/java/org/apache/lucene/facet/sampling/TakmiSampleFixer.java (working copy) @@ -2,21 +2,19 @@ import java.io.IOException; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.Bits; - import org.apache.lucene.facet.params.FacetSearchParams; import org.apache.lucene.facet.search.DrillDownQuery; -import org.apache.lucene.facet.search.FacetResult; import org.apache.lucene.facet.search.FacetResultNode; import org.apache.lucene.facet.search.ScoredDocIDs; import org.apache.lucene.facet.search.ScoredDocIDsIterator; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.Bits; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -36,16 +34,22 @@ */ /** - * Fix sampling results by counting the intersection between two lists: a - * TermDocs (list of documents in a certain category) and a DocIdSetIterator - * (list of documents matching the query). + * Fix sampling results by correct results, by counting the intersection between + * two lists: a TermDocs (list of documents in a certain category) and a + * DocIdSetIterator (list of documents matching the query). + *

+ * This fixer is suitable for scenarios which prioritize accuracy over + * performance. For scenarios prioritizing performance over accuracy, see + * {@link AmortizedSampleFixer} + *

+ * Note: for statistically more accurate top-k selection, set + * {@link SamplingParams#setOversampleFactor(double) oversampleFactor} to at + * least 2, so that the top-k categories would have better chance of showing up + * in the sampled top-cK results (see {@link SamplingParams#getOversampleFactor} * - * * @lucene.experimental */ -// TODO (Facet): implement also an estimated fixing by ratio (taking into -// account "translation" of counts!) -class TakmiSampleFixer implements SampleFixer { +class TakmiSampleFixer extends SampleFixer { private TaxonomyReader taxonomyReader; private IndexReader indexReader; @@ -59,29 +63,11 @@ } @Override - public void fixResult(ScoredDocIDs origDocIds, FacetResult fres) - throws IOException { - FacetResultNode topRes = fres.getFacetResultNode(); - fixResultNode(topRes, origDocIds); + public void singleNodeFix(FacetResultNode facetResNode, ScoredDocIDs docIds, double samplingRatio) throws IOException { + recount(facetResNode, docIds); } /** - * Fix result node count, and, recursively, fix all its children - * - * @param facetResNode - * result node to be fixed - * @param docIds - * docids in effect - * @throws IOException If there is a low-level I/O error. - */ - private void fixResultNode(FacetResultNode facetResNode, ScoredDocIDs docIds) throws IOException { - recount(facetResNode, docIds); - for (FacetResultNode frn : facetResNode.subResults) { - fixResultNode(frn, docIds); - } - } - - /** * Internal utility: recount for a facet result node * * @param fresNode @@ -179,4 +165,5 @@ } return false; // exhausted } + } \ No newline at end of file Index: lucene/facet/src/test/org/apache/lucene/facet/sampling/AmortizedSamplerFixerTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/sampling/AmortizedSamplerFixerTest.java (revision 0) +++ lucene/facet/src/test/org/apache/lucene/facet/sampling/AmortizedSamplerFixerTest.java (working copy) @@ -0,0 +1,55 @@ +package org.apache.lucene.facet.sampling; + +import java.util.ArrayList; + +import org.apache.lucene.facet.FacetTestBase; +import org.apache.lucene.facet.search.CountFacetRequest; +import org.apache.lucene.facet.search.FacetResult; +import org.apache.lucene.facet.search.FacetResultNode; +import org.apache.lucene.facet.taxonomy.CategoryPath; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class AmortizedSamplerFixerTest extends FacetTestBase { + + public void testFixing() throws Exception { + // a random sample ratio at the range of [0.1, 0.6) + final double sampleRatio = 0.1d + (random().nextDouble() / 2d); + + AmortizedSampleFixer asf = new AmortizedSampleFixer(); + FacetResultNode root = new FacetResultNode(1, 11.0d); + final FacetResultNode child1 = new FacetResultNode(2, 7.0d); + final FacetResultNode child2 = new FacetResultNode(3, 4.0d); + final FacetResultNode child3 = new FacetResultNode(4, 3.0d); + root.subResults = new ArrayList() {{ + add(child1); + add(child2); + add(child3); + }}; + + FacetResult res = new FacetResult(new CountFacetRequest(new CategoryPath("path"), 10), root, 10); + asf.fixResult(null, res, sampleRatio); + + assertEquals(11.0 / sampleRatio, root.value, Double.MIN_NORMAL); + assertEquals(7.0 / sampleRatio, child1.value, Double.MIN_NORMAL); + assertEquals(4.0 / sampleRatio, child2.value, Double.MIN_NORMAL); + assertEquals(3.0 / sampleRatio, child3.value, Double.MIN_NORMAL); + } + + +} Index: lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java (revision 1486398) +++ lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java (working copy) @@ -94,7 +94,7 @@ for (int nTrial = 0; nTrial < RETRIES; nTrial++) { try { // complement with sampling! - final Sampler sampler = createSampler(nTrial, useRandomSampler); + final Sampler sampler = createSampler(nTrial, useRandomSampler, samplingSearchParams); assertSampling(expectedResults, q, sampler, samplingSearchParams, false); assertSampling(expectedResults, q, sampler, samplingSearchParams, true); @@ -128,14 +128,20 @@ return FacetsCollector.create(sfa); } - private Sampler createSampler(int nTrial, boolean useRandomSampler) { + private Sampler createSampler(int nTrial, boolean useRandomSampler, FacetSearchParams sParams) { SamplingParams samplingParams = new SamplingParams(); + /* + * Set sampling to Exact fixing with TakmiSampleFixer as it is not easy to + * validate results with amortized results. + */ + samplingParams.setSampleFixer(new TakmiSampleFixer(indexReader, taxoReader, sParams)); + final double retryFactor = Math.pow(1.01, nTrial); + samplingParams.setOversampleFactor(5.0 * retryFactor); // Oversampling samplingParams.setSampleRatio(0.8 * retryFactor); samplingParams.setMinSampleSize((int) (100 * retryFactor)); samplingParams.setMaxSampleSize((int) (10000 * retryFactor)); - samplingParams.setOversampleFactor(5.0 * retryFactor); samplingParams.setSamplingThreshold(11000); //force sampling Sampler sampler = useRandomSampler ? Index: lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplerTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplerTest.java (revision 0) +++ lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplerTest.java (working copy) @@ -0,0 +1,36 @@ +package org.apache.lucene.facet.sampling; + +import org.apache.lucene.facet.FacetTestBase; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class SamplerTest extends FacetTestBase { + + public void testDefaultFixer() throws Exception { + RandomSampler randomSampler = new RandomSampler(); + SampleFixer fixer = randomSampler.samplingParams.getSampleFixer(); + assertEquals(AmortizedSampleFixer.class, fixer.getClass()); + } + + public void testCustomFixer() throws Exception { + SamplingParams sp = new SamplingParams(); + sp.setSampleFixer(new SampleFixer.NoopSampleFixer()); + assertEquals(SampleFixer.NoopSampleFixer.class, sp.getSampleFixer().getClass()); + } + +}