Index: lucene/facet/src/java/org/apache/lucene/facet/associations/SumFloatAssociationFacetsAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/SumFloatAssociationFacetsAggregator.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/SumFloatAssociationFacetsAggregator.java (working copy) @@ -9,6 +9,7 @@ import org.apache.lucene.facet.search.FacetsCollector.MatchingDocs; import org.apache.lucene.facet.search.OrdinalValueResolver; import org.apache.lucene.facet.search.OrdinalValueResolver.FloatValueResolver; +import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.BytesRef; @@ -35,7 +36,7 @@ * the association encoded for each ordinal is {@link CategoryFloatAssociation}. *

* NOTE: this aggregator does not support - * {@link #rollupValues(FacetRequest, int, int[], int[], FacetArrays)}. It only + * {@link #rollupValues(FacetRequest, int, TaxonomyReader, FacetArrays)}. It only * aggregates the categories for which you added a {@link CategoryAssociation}. * * @lucene.experimental @@ -80,7 +81,7 @@ } @Override - public void rollupValues(FacetRequest fr, int ordinal, int[] children, int[] siblings, FacetArrays facetArrays) { + public void rollupValues(FacetRequest fr, int ordinal, TaxonomyReader taxonomyReader, FacetArrays facetArrays) throws IOException { // NO-OP: this aggregator does no rollup values to the parents. } Index: lucene/facet/src/java/org/apache/lucene/facet/associations/SumIntAssociationFacetsAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/SumIntAssociationFacetsAggregator.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/SumIntAssociationFacetsAggregator.java (working copy) @@ -9,6 +9,7 @@ import org.apache.lucene.facet.search.FacetsCollector.MatchingDocs; import org.apache.lucene.facet.search.OrdinalValueResolver; import org.apache.lucene.facet.search.OrdinalValueResolver.IntValueResolver; +import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.BytesRef; @@ -36,7 +37,7 @@ * {@link CategoryIntAssociation}. *

* NOTE: this aggregator does not support - * {@link #rollupValues(FacetRequest, int, int[], int[], FacetArrays)}. It only + * {@link #rollupValues(FacetRequest, int, TaxonomyReader, FacetArrays)}. It only * aggregates the categories for which you added a {@link CategoryAssociation}. */ public class SumIntAssociationFacetsAggregator implements FacetsAggregator { @@ -79,7 +80,7 @@ } @Override - public void rollupValues(FacetRequest fr, int ordinal, int[] children, int[] siblings, FacetArrays facetArrays) { + public void rollupValues(FacetRequest fr, int ordinal, TaxonomyReader taxonomyReader, FacetArrays facetArrays) throws IOException { // NO-OP: this aggregator does no rollup values to the parents. } Index: lucene/facet/src/java/org/apache/lucene/facet/search/DepthOneFacetResultsHandler.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/DepthOneFacetResultsHandler.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/DepthOneFacetResultsHandler.java (working copy) @@ -7,7 +7,6 @@ import java.util.Comparator; import org.apache.lucene.facet.search.FacetRequest.SortOrder; -import org.apache.lucene.facet.taxonomy.ParallelTaxonomyArrays; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.PriorityQueue; @@ -66,26 +65,28 @@ @Override public final FacetResult compute() throws IOException { - ParallelTaxonomyArrays arrays = taxonomyReader.getParallelTaxonomyArrays(); - final int[] children = arrays.children(); - final int[] siblings = arrays.siblings(); + int rootOrd = taxonomyReader.getOrdinal(facetRequest.categoryPath); - int rootOrd = taxonomyReader.getOrdinal(facetRequest.categoryPath); - FacetResultNode root = new FacetResultNode(rootOrd, resolver.valueOf(rootOrd)); root.label = facetRequest.categoryPath; + + int[] children = taxonomyReader.getChildren(rootOrd); + if (children == TaxonomyReader.NO_CHILDREN) { + return new FacetResult(facetRequest, root, 0); + } + if (facetRequest.numResults > taxonomyReader.getSize()) { // specialize this case, user is interested in all available results ArrayList nodes = new ArrayList(); - int ordinal = children[rootOrd]; - while (ordinal != TaxonomyReader.INVALID_ORDINAL) { + final int end = children.length-1; + for(int i = 0; i < end; ++i) { + int ordinal = children[i]; double value = resolver.valueOf(ordinal); if (value > 0) { FacetResultNode node = new FacetResultNode(ordinal, value); node.label = taxonomyReader.getPath(ordinal); nodes.add(node); } - ordinal = siblings[ordinal]; } CollectionUtil.introSort(nodes, Collections.reverseOrder(new Comparator() { @@ -101,10 +102,11 @@ // since we use sentinel objects, we cannot reuse PQ. but that's ok because it's not big PriorityQueue pq = new FacetResultNodeQueue(facetRequest.numResults, true); - int ordinal = children[rootOrd]; FacetResultNode top = pq.top(); int numSiblings = 0; - while (ordinal != TaxonomyReader.INVALID_ORDINAL) { + final int end = children.length-1; + for ( int i = 0; i < end; ++i) { + int ordinal = children[i]; double value = resolver.valueOf(ordinal); if (value > 0) { ++numSiblings; @@ -114,7 +116,6 @@ top = pq.updateTop(); } } - ordinal = siblings[ordinal]; } // pop() the least (sentinel) elements Index: lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAggregator.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAggregator.java (working copy) @@ -5,6 +5,7 @@ import org.apache.lucene.facet.params.CategoryListParams; import org.apache.lucene.facet.params.CategoryListParams.OrdinalPolicy; import org.apache.lucene.facet.search.FacetsCollector.MatchingDocs; +import org.apache.lucene.facet.taxonomy.TaxonomyReader; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -38,10 +39,10 @@ /** * Rollup the values of the given ordinal. This method is called when a * category was indexed with {@link OrdinalPolicy#NO_PARENTS}. The given - * ordinal is the requested category, and you should use the children and - * siblings arrays to traverse its sub-tree. + * ordinal is the requested category, and you should use the + * {@link TaxonomyReader#getChildren(int)} to traverse its sub-tree. */ - public void rollupValues(FacetRequest fr, int ordinal, int[] children, int[] siblings, FacetArrays facetArrays); + public void rollupValues(FacetRequest fr, int ordinal, TaxonomyReader taxoReader, FacetArrays facetArrays) throws IOException; /** Returns {@code true} if this aggregator requires document scores. */ public boolean requiresDocScores(); Index: lucene/facet/src/java/org/apache/lucene/facet/search/IntRollupFacetsAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/IntRollupFacetsAggregator.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/IntRollupFacetsAggregator.java (working copy) @@ -26,7 +26,7 @@ /** * A {@link FacetsAggregator} which implements - * {@link #rollupValues(FacetRequest, int, int[], int[], FacetArrays)} by + * {@link #rollupValues(FacetRequest, int, TaxonomyReader, FacetArrays)} by * summing the values from {@link FacetArrays#getIntArray()}. Extending classes * should only implement {@link #aggregate}. Also, {@link #requiresDocScores()} * always returns false. @@ -38,22 +38,27 @@ @Override public abstract void aggregate(MatchingDocs matchingDocs, CategoryListParams clp, FacetArrays facetArrays) throws IOException; - private int rollupValues(int ordinal, int[] children, int[] siblings, int[] values) { + private int rollupValues(int[] childrenIterator, TaxonomyReader taxoReader, int[] values) throws IOException { int value = 0; - while (ordinal != TaxonomyReader.INVALID_ORDINAL) { + final int end = childrenIterator.length-1; + for ( int i = 0; i < end; ++i) { + int ordinal = childrenIterator[i]; int childValue = values[ordinal]; - childValue += rollupValues(children[ordinal], children, siblings, values); + int[] children = taxoReader.getChildren(ordinal); + if (children != TaxonomyReader.NO_CHILDREN) { + childValue += rollupValues(children, taxoReader, values); + } values[ordinal] = childValue; value += childValue; - ordinal = siblings[ordinal]; + } return value; } @Override - public final void rollupValues(FacetRequest fr, int ordinal, int[] children, int[] siblings, FacetArrays facetArrays) { + public final void rollupValues(FacetRequest fr, int ordinal, TaxonomyReader taxoReader, FacetArrays facetArrays) throws IOException { final int[] values = facetArrays.getIntArray(); - values[ordinal] += rollupValues(children[ordinal], children, siblings, values); + values[ordinal] += rollupValues(taxoReader.getChildren(ordinal), taxoReader, values); } @Override Index: lucene/facet/src/java/org/apache/lucene/facet/search/MultiFacetsAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/MultiFacetsAggregator.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/MultiFacetsAggregator.java (working copy) @@ -7,11 +7,9 @@ import java.util.Map; import org.apache.lucene.facet.params.CategoryListParams; -import org.apache.lucene.facet.search.FacetArrays; -import org.apache.lucene.facet.search.FacetRequest; -import org.apache.lucene.facet.search.FacetsAggregator; import org.apache.lucene.facet.search.FacetsCollector.MatchingDocs; import org.apache.lucene.facet.taxonomy.CategoryPath; +import org.apache.lucene.facet.taxonomy.TaxonomyReader; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -74,8 +72,8 @@ } @Override - public void rollupValues(FacetRequest fr, int ordinal, int[] children, int[] siblings, FacetArrays facetArrays) { - categoryAggregators.get(fr.categoryPath).rollupValues(fr, ordinal, children, siblings, facetArrays); + public void rollupValues(FacetRequest fr, int ordinal, TaxonomyReader taxoReader, FacetArrays facetArrays) throws IOException { + categoryAggregators.get(fr.categoryPath).rollupValues(fr, ordinal, taxoReader, facetArrays); } @Override Index: lucene/facet/src/java/org/apache/lucene/facet/search/PerCategoryListAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/PerCategoryListAggregator.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/PerCategoryListAggregator.java (working copy) @@ -6,6 +6,7 @@ import org.apache.lucene.facet.params.CategoryListParams; import org.apache.lucene.facet.params.FacetIndexingParams; import org.apache.lucene.facet.search.FacetsCollector.MatchingDocs; +import org.apache.lucene.facet.taxonomy.TaxonomyReader; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -27,7 +28,7 @@ /** * A {@link FacetsAggregator} which invokes the proper aggregator per * {@link CategoryListParams}. - * {@link #rollupValues(FacetRequest, int, int[], int[], FacetArrays)} is + * {@link #rollupValues(FacetRequest, int, TaxonomyReader, FacetArrays)} is * delegated to the proper aggregator which handles the * {@link CategoryListParams} the given {@link FacetRequest} belongs to. */ @@ -47,9 +48,9 @@ } @Override - public void rollupValues(FacetRequest fr, int ordinal, int[] children, int[] siblings, FacetArrays facetArrays) { + public void rollupValues(FacetRequest fr, int ordinal, TaxonomyReader taxoReader, FacetArrays facetArrays) throws IOException { CategoryListParams clp = fip.getCategoryListParams(fr.categoryPath); - aggregators.get(clp).rollupValues(fr, ordinal, children, siblings, facetArrays); + aggregators.get(clp).rollupValues(fr, ordinal, taxoReader, facetArrays); } @Override Index: lucene/facet/src/java/org/apache/lucene/facet/search/SumScoreFacetsAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/SumScoreFacetsAggregator.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/SumScoreFacetsAggregator.java (working copy) @@ -55,22 +55,24 @@ } } - private float rollupScores(int ordinal, int[] children, int[] siblings, float[] scores) { + private float rollupScores(int[] children, TaxonomyReader taxoReader, float[] scores) throws IOException { float score = 0f; + int i = 0; + int ordinal = children[i++]; while (ordinal != TaxonomyReader.INVALID_ORDINAL) { float childScore = scores[ordinal]; - childScore += rollupScores(children[ordinal], children, siblings, scores); + childScore += rollupScores(taxoReader.getChildren(ordinal), taxoReader, scores); scores[ordinal] = childScore; score += childScore; - ordinal = siblings[ordinal]; + ordinal = children[i++]; } return score; } @Override - public void rollupValues(FacetRequest fr, int ordinal, int[] children, int[] siblings, FacetArrays facetArrays) { + public void rollupValues(FacetRequest fr, int ordinal, TaxonomyReader taxoReader, FacetArrays facetArrays) throws IOException { float[] scores = facetArrays.getFloatArray(); - scores[ordinal] += rollupScores(children[ordinal], children, siblings, scores); + scores[ordinal] += rollupScores(taxoReader.getChildren(ordinal), taxoReader, scores); } @Override Index: lucene/facet/src/java/org/apache/lucene/facet/search/SumValueSourceFacetRequest.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/SumValueSourceFacetRequest.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/SumValueSourceFacetRequest.java (working copy) @@ -54,22 +54,30 @@ this.valueSource = valueSource; } - private float doRollup(int ordinal, int[] children, int[] siblings, float[] values) { + private float doRollup(int[] kids, TaxonomyReader taxo, float[] values) throws IOException { float value = 0f; + int i = 0; + int ordinal = kids[i++]; while (ordinal != TaxonomyReader.INVALID_ORDINAL) { float childValue = values[ordinal]; - childValue += doRollup(children[ordinal], children, siblings, values); + int[] children = taxo.getChildren(ordinal); + if (children != TaxonomyReader.NO_CHILDREN) { + childValue += doRollup(children, taxo, values); + } values[ordinal] = childValue; value += childValue; - ordinal = siblings[ordinal]; + ordinal = kids[i++]; } return value; } @Override - public void rollupValues(FacetRequest fr, int ordinal, int[] children, int[] siblings, FacetArrays facetArrays) { + public void rollupValues(FacetRequest fr, int ordinal, TaxonomyReader taxoReader, FacetArrays facetArrays) throws IOException { float[] values = facetArrays.getFloatArray(); - values[ordinal] += doRollup(children[ordinal], children, siblings, values); + int[] children = taxoReader.getChildren(ordinal); + if (children != TaxonomyReader.NO_CHILDREN) { + values[ordinal] += doRollup(children, taxoReader, values); + } } @Override Index: lucene/facet/src/java/org/apache/lucene/facet/search/TaxonomyFacetsAccumulator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/TaxonomyFacetsAccumulator.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/TaxonomyFacetsAccumulator.java (working copy) @@ -15,7 +15,6 @@ import org.apache.lucene.facet.search.FacetRequest.SortOrder; import org.apache.lucene.facet.search.FacetsCollector.MatchingDocs; import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.ParallelTaxonomyArrays; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.index.IndexReader; @@ -183,11 +182,7 @@ } } - ParallelTaxonomyArrays arrays = taxonomyReader.getParallelTaxonomyArrays(); - // compute top-K - final int[] children = arrays.children(); - final int[] siblings = arrays.siblings(); List res = new ArrayList(); for (FacetRequest fr : searchParams.facetRequests) { int rootOrd = taxonomyReader.getOrdinal(fr.categoryPath); @@ -201,7 +196,7 @@ OrdinalPolicy ordinalPolicy = clp.getOrdinalPolicy(fr.categoryPath.components[0]); if (ordinalPolicy == OrdinalPolicy.NO_PARENTS) { // rollup values - aggregator.rollupValues(fr, rootOrd, children, siblings, facetArrays); + aggregator.rollupValues(fr, rootOrd, taxonomyReader, facetArrays); } } Index: lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/TopKFacetResultsHandler.java (working copy) @@ -5,7 +5,6 @@ import org.apache.lucene.facet.partitions.IntermediateFacetResult; import org.apache.lucene.facet.partitions.PartitionsFacetResultsHandler; -import org.apache.lucene.facet.taxonomy.ParallelTaxonomyArrays; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.facet.util.ResultSortUtils; @@ -96,6 +95,16 @@ return res; } + private static class ChildrenHolder { + private int idx; + private final int[] children; + + public ChildrenHolder(int[] children) { + this.children = children; + this.idx = 0; + } + } + /** * Finds the top K descendants of ordinal, which are at most facetRequest.getDepth() * deeper than facetRequest.getCategoryPath (whose ordinal is input parameter ordinal). @@ -107,24 +116,25 @@ int offset) throws IOException { int partitionSize = facetArrays.arrayLength; int endOffset = offset + partitionSize; - ParallelTaxonomyArrays childrenArray = taxonomyReader.getParallelTaxonomyArrays(); - int[] children = childrenArray.children(); - int[] siblings = childrenArray.siblings(); FacetResultNode reusable = null; int localDepth = 0; int depth = facetRequest.getDepth(); int[] ordinalStack = new int[2+Math.min(Short.MAX_VALUE, depth)]; int childrenCounter = 0; - + ChildrenHolder[] childStack = new ChildrenHolder[ordinalStack.length]; int tosOrdinal; // top of stack element - int yc = children[ordinal]; + childStack[0] = new ChildrenHolder(TaxonomyReader.NO_CHILDREN); + + ChildrenHolder kids = new ChildrenHolder(taxonomyReader.getChildren(ordinal)); + int yc = kids.children[kids.idx++]; while (yc >= endOffset) { - yc = siblings[yc]; + yc = kids.children[kids.idx++]; } // make use of the fact that TaxonomyReader.INVALID_ORDINAL == -1, < endOffset // and it, too, can stop the loop. ordinalStack[++localDepth] = yc; + childStack[localDepth] = kids; /* * stack holds input parameter ordinal in position 0. @@ -143,9 +153,13 @@ if (tosOrdinal == TaxonomyReader.INVALID_ORDINAL) { // element below tos has all its children, and itself, all processed // need to proceed to its sibling + if (localDepth == 1) { + break; + } localDepth--; // change element now on top of stack to its sibling. - ordinalStack[localDepth] = siblings[ordinalStack[localDepth]]; + ChildrenHolder ch = childStack[localDepth]; + ordinalStack[localDepth] = ch.children[ch.idx++]; continue; } // top of stack is not invalid, this is the first time we see it on top of stack. @@ -168,13 +182,16 @@ reusable = pq.insertWithOverflow(reusable); } } + // push next kid to the stack if (localDepth < depth) { - // push kid of current tos - yc = children[tosOrdinal]; + kids = new ChildrenHolder(taxonomyReader.getChildren(tosOrdinal)); + yc = kids.children[kids.idx++]; while (yc >= endOffset) { - yc = siblings[yc]; + yc = kids.children[kids.idx++]; } + ordinalStack[++localDepth] = yc; + childStack[localDepth] = kids; } else { // localDepth == depth; current tos exhausted its possible children, mark this by pushing INVALID_ORDINAL ordinalStack[++localDepth] = TaxonomyReader.INVALID_ORDINAL; } Index: lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/search/TopKInEachNodeHandler.java (working copy) @@ -9,7 +9,6 @@ import org.apache.lucene.facet.partitions.IntermediateFacetResult; import org.apache.lucene.facet.partitions.PartitionsFacetResultsHandler; import org.apache.lucene.facet.search.FacetRequest.SortOrder; -import org.apache.lucene.facet.taxonomy.ParallelTaxonomyArrays; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.util.PriorityQueue; @@ -66,7 +65,16 @@ FacetArrays facetArrays) { super(taxonomyReader, facetRequest, resolver, facetArrays); } + + private static class ChildrenHolder { + private int idx; + private final int[] children; + public ChildrenHolder(int[] children) { + this.children = children; + this.idx = 0; + } + } /** * Recursively explore all facets that can be potentially included in the * {@link FacetResult} to be generated, and that belong to the given @@ -134,9 +142,6 @@ } int endOffset = offset + partitionSize; // one past the largest ordinal in the partition - ParallelTaxonomyArrays childrenArray = taxonomyReader.getParallelTaxonomyArrays(); - int[] children = childrenArray.children(); - int[] siblings = childrenArray.siblings(); int totalNumOfDescendantsConsidered = 0; // total number of facets with value != 0, // in the tree. These include those selected as top K in each node, and all the others that // were not. Not including rootNode @@ -198,6 +203,7 @@ int[][] bestSignlingsStack = new int[depth+2][]; int[] siblingExplored = new int[depth+2]; int[] firstToTheLeftOfPartition = new int [depth+2]; + ChildrenHolder[] childStack = new ChildrenHolder[depth+2]; int tosOrdinal; // top of stack element, the ordinal at the top of stack @@ -210,7 +216,9 @@ * we can continue to the older sibling of rootNode once the localDepth goes down, before we verify that * it went that down) */ - ordinalStack[++localDepth] = children[rootNode]; + ChildrenHolder kids = new ChildrenHolder(taxonomyReader.getChildren(rootNode)); + ordinalStack[++localDepth] = kids.children[kids.idx++]; + childStack[localDepth] = kids; siblingExplored[localDepth] = Integer.MAX_VALUE; // we have not verified position wrt current partition siblingExplored[0] = -1; // as if rootNode resides to the left of current position @@ -222,16 +230,23 @@ while (localDepth > 0) { tosOrdinal = ordinalStack[localDepth]; if (tosOrdinal == TaxonomyReader.INVALID_ORDINAL) { + // element below tos has all its children, and itself, all processed + // need to proceed to its sibling + if (localDepth == 1) { + break; + } // the brotherhood that has been occupying the top of stack is all exhausted. // Hence, element below tos, namely, father of tos, has all its children, // and itself, all explored. localDepth--; + // replace this father, now on top of stack, by this father's sibling: // this parent's ordinal can not be greater than current partition, as otherwise // its child, now just removed, would not have been pushed on it. // so the father is either inside the partition, or smaller ordinal if (siblingExplored[localDepth] < 0 ) { - ordinalStack[localDepth] = siblings[ordinalStack[localDepth]]; + ChildrenHolder ch = childStack[localDepth]; + ordinalStack[localDepth] = ch.children[ch.idx++]; continue; } // in this point, siblingExplored[localDepth] between 0 and number of bestSiblings @@ -257,7 +272,8 @@ //tosOrdinal was not examined yet for its position relative to current partition // and the best K of current partition, among its siblings, have not been determined yet while (tosOrdinal >= endOffset) { - tosOrdinal = siblings[tosOrdinal]; + ChildrenHolder ch = childStack[localDepth]; + tosOrdinal = ch.children[ch.idx++]; } // now it is inside. Run it and all its siblings inside the partition through a heap // and in doing so, count them, find best K @@ -284,12 +300,12 @@ // update totalNumOfDescendants by the now excluded node and all its descendants totalNumOfDescendantsConsidered--; // reduce the 1 earned when the excluded node entered the heap // and now return it and all its descendants. These will never make it to FacetResult - totalNumOfDescendantsConsidered += countOnly (ac.ordinal, children, - siblings, partitionSize, offset, endOffset, localDepth, depth); + totalNumOfDescendantsConsidered += countOnly (ac.ordinal, partitionSize, offset, endOffset, localDepth, depth); reusables[++tosReuslables] = ac; } } - tosOrdinal = siblings[tosOrdinal]; + ChildrenHolder ch = childStack[localDepth]; + tosOrdinal = ch.children[ch.idx++]; } // now pq has best K children of ordinals that belong to the given partition. // Populate a new AACO with them. @@ -330,7 +346,11 @@ ordinalStack[++localDepth] = TaxonomyReader.INVALID_ORDINAL; continue; } - ordinalStack[++localDepth] = children[tosOrdinal]; + + kids = new ChildrenHolder(taxonomyReader.getChildren(tosOrdinal)); + ordinalStack[++localDepth] = kids.children[kids.idx++]; + childStack[localDepth] = kids; + siblingExplored[localDepth] = Integer.MAX_VALUE; } // endof loop while stack is not empty @@ -358,9 +378,6 @@ * as ordinal's descendants might be >= offeset. * * @param ordinal a facet ordinal. - * @param youngestChild mapping a given ordinal to its youngest child in the taxonomy (of largest ordinal number), - * or to -1 if has no children. - * @param olderSibling mapping a given ordinal to its older sibling, or to -1 * @param partitionSize number of ordinals in the given partition * @param offset the first (smallest) ordinal in the given partition * @param endOffset one larger than the largest ordinal that belong to this partition @@ -370,8 +387,8 @@ * @return the number of nodes, from ordinal down its descendants, of depth <= maxDepth, * which reside in the current partition, and whose value != 0 */ - private int countOnly(int ordinal, int[] youngestChild, int[] olderSibling, int partitionSize, int offset, - int endOffset, int currentDepth, int maxDepth) { + private int countOnly(int ordinal, int partitionSize, int offset, + int endOffset, int currentDepth, int maxDepth) throws IOException { int ret = 0; if (offset <= ordinal) { // ordinal belongs to the current partition @@ -384,14 +401,15 @@ return ret; } - int yc = youngestChild[ordinal]; + ChildrenHolder kids = new ChildrenHolder(taxonomyReader.getChildren(ordinal)); + int yc = kids.children[kids.idx++]; while (yc >= endOffset) { - yc = olderSibling[yc]; + yc = kids.children[kids.idx++]; } while (yc > TaxonomyReader.INVALID_ORDINAL) { // assuming this is -1, smaller than any legal ordinal - ret += countOnly (yc, youngestChild, olderSibling, partitionSize, + ret += countOnly (yc, partitionSize, offset, endOffset, currentDepth+1, maxDepth); - yc = olderSibling[yc]; + yc = kids.children[kids.idx++]; } return ret; } Index: lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java (working copy) @@ -71,7 +71,7 @@ private final int[] siblings; private int child; - ChildrenIterator(int child, int[] siblings) { + public ChildrenIterator(int child, int[] siblings) { this.siblings = siblings; this.child = child; } @@ -104,6 +104,7 @@ */ public final static int INVALID_ORDINAL = -1; + public final static int[] NO_CHILDREN = new int[] {INVALID_ORDINAL}; /** * If the taxonomy has changed since the provided reader was opened, open and * return a new {@link TaxonomyReader}; else, return {@code null}. The new @@ -190,14 +191,10 @@ * Returns a {@link ParallelTaxonomyArrays} object which can be used to * efficiently traverse the taxonomy tree. */ - public abstract ParallelTaxonomyArrays getParallelTaxonomyArrays() throws IOException; + protected abstract ParallelTaxonomyArrays getParallelTaxonomyArrays() throws IOException; /** Returns an iterator over the children of the given ordinal. */ - public ChildrenIterator getChildren(final int ordinal) throws IOException { - ParallelTaxonomyArrays arrays = getParallelTaxonomyArrays(); - int child = ordinal >= 0 ? arrays.children()[ordinal] : INVALID_ORDINAL; - return new ChildrenIterator(child, arrays.siblings()); - } + public abstract int[] getChildren(final int ordinal) throws IOException ; /** * Retrieve user committed data. Index: lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java (working copy) @@ -1,6 +1,8 @@ package org.apache.lucene.facet.taxonomy.directory; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -227,13 +229,21 @@ } @Override - public ParallelTaxonomyArrays getParallelTaxonomyArrays() throws IOException { + protected ParallelTaxonomyArrays getParallelTaxonomyArrays() throws IOException { ensureOpen(); if (taxoArrays == null) { initTaxoArrays(); } return taxoArrays; } + + protected HashMap getChildrenMap() throws IOException { + ensureOpen(); + if (children == null) { + initChildrenMap(); + } + return children; + } @Override public Map getCommitUserData() throws IOException { @@ -382,5 +392,57 @@ } return sb.toString(); } + + // nocommit ---- + // A map-based childrenIterator + // it does not support reopen using old data (reopen is at O(TaxoSize) and not O(Change size)) + private volatile HashMap children = null; + private synchronized void initChildrenMap() throws IOException { + if (children == null) { + HashMap newChildren = new HashMap(); + for (int ordinal = 0; ordinal < getSize() ; ++ordinal) { + ChildrenIterator taxoChildren = getChildrenOld(ordinal); + if (taxoChildren == null) { + continue; + } + ArrayList kids = new ArrayList(); + int kidOrdinal; + while((kidOrdinal = taxoChildren.next()) != TaxonomyReader.INVALID_ORDINAL) { + kids.add(kidOrdinal); + } + int i = 0; + int[] intKids = new int[kids.size() + 1]; + for (Integer integer : kids) { + intKids[i++] = integer.intValue(); + } + // Adding last child as invalid. It wastes an int per (real) parent in + // memory, but spares an if call for every next(), avoiding AIOOB + intKids[i] = TaxonomyReader.INVALID_ORDINAL; + newChildren.put(ordinal, intKids); + } + children = newChildren; + } + } + @Override + public int[] getChildren(int ordinal) throws IOException { + HashMap childrenMap = getChildrenMap(); + + final int[] kids = childrenMap.get(ordinal); + if (kids == null) { + return NO_CHILDREN; + } + return kids; + } + + /** Returns an iterator over the children of the given ordinal. */ + public ChildrenIterator getChildrenOld(final int ordinal) throws IOException { + ParallelTaxonomyArrays arrays = getParallelTaxonomyArrays(); + int child = ordinal >= 0 ? arrays.children()[ordinal] : INVALID_ORDINAL; + if (child >=0) { + return new ChildrenIterator(child, arrays.siblings()); + } + return null; + } + } Index: lucene/facet/src/java/org/apache/lucene/facet/util/PrintTaxonomyStats.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/util/PrintTaxonomyStats.java (revision 1542799) +++ lucene/facet/src/java/org/apache/lucene/facet/util/PrintTaxonomyStats.java (working copy) @@ -23,7 +23,6 @@ import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -import org.apache.lucene.facet.taxonomy.TaxonomyReader.ChildrenIterator; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -57,14 +56,16 @@ public static void printStats(TaxonomyReader r, PrintStream out, boolean printTree) throws IOException { out.println(r.getSize() + " total categories."); - ChildrenIterator it = r.getChildren(TaxonomyReader.ROOT_ORDINAL); + int[] kids = r.getChildren(TaxonomyReader.ROOT_ORDINAL); int child; - while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) { - ChildrenIterator chilrenIt = r.getChildren(child); - int numImmediateChildren = 0; - while (chilrenIt.next() != TaxonomyReader.INVALID_ORDINAL) { - numImmediateChildren++; - } + int i = 0; + while ((child = kids[i++]) != TaxonomyReader.INVALID_ORDINAL) { + int[] chilrenIt = r.getChildren(child); + int numImmediateChildren = chilrenIt.length-1; +// int j = 0; +// while (chilrenIt[j++] != TaxonomyReader.INVALID_ORDINAL) { +// numImmediateChildren++; +// } CategoryPath cp = r.getPath(child); out.println("/" + cp + ": " + numImmediateChildren + " immediate children; " + (1+countAllChildren(r, child)) + " total categories"); if (printTree) { @@ -74,21 +75,29 @@ } private static int countAllChildren(TaxonomyReader r, int ord) throws IOException { + int[] it = r.getChildren(ord); + if (it == TaxonomyReader.NO_CHILDREN) { + return 0; + } + int count = 0; - ChildrenIterator it = r.getChildren(ord); int child; - while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) { + int j = 0; + while ((child = it[j++]) != TaxonomyReader.INVALID_ORDINAL) { count += 1 + countAllChildren(r, child); } return count; } private static void printAllChildren(PrintStream out, TaxonomyReader r, int ord, String indent, int depth) throws IOException { - ChildrenIterator it = r.getChildren(ord); + int[] it = r.getChildren(ord); int child; - while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) { - out.println(indent + "/" + r.getPath(child).components[depth]); - printAllChildren(out, r, child, indent + " ", depth+1); + int i = 0; + if (it != TaxonomyReader.NO_CHILDREN) { + while ((child = it[i++]) != TaxonomyReader.INVALID_ORDINAL) { + out.println(indent + "/" + r.getPath(child).components[depth]); + printAllChildren(out, r, child, indent + " ", depth+1); + } } } } Index: lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java (revision 1542799) +++ lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyCombined.java (working copy) @@ -769,9 +769,9 @@ setPriority(1 + getPriority()); try { while (!stop.get()) { - int lastOrd = tr.getParallelTaxonomyArrays().parents().length - 1; + int lastOrd = tr.getSize() - 1; assertNotNull("path of last-ord " + lastOrd + " is not found!", tr.getPath(lastOrd)); - assertChildrenArrays(tr.getParallelTaxonomyArrays(), retry, retrieval[0]++); + assertChildrenArrays(tr, retry, retrieval[0]++); sleep(10); // don't starve refresh()'s CPU, which sleeps every 50 bytes for 1 ms } } catch (Throwable e) { @@ -780,13 +780,15 @@ } } - private void assertChildrenArrays(ParallelTaxonomyArrays ca, int retry, int retrieval) { - final int abYoungChild = ca.children()[abOrd]; + private void assertChildrenArrays(TaxonomyReader tr, int retry, int retrieval) throws IOException { + int[] children = tr.getChildren(abOrd); + + final int abYoungChild = children[0]; assertTrue( "Retry "+retry+": retrieval: "+retrieval+": wrong youngest child for category "+abPath+" (ord="+abOrd+ ") - must be either "+abYoungChildBase1+" or "+abYoungChildBase2+" but was: "+abYoungChild, abYoungChildBase1==abYoungChild || - abYoungChildBase2==ca.children()[abOrd]); + abYoungChildBase2==(children != TaxonomyReader.NO_CHILDREN ? children[1] : TaxonomyReader.INVALID_ORDINAL)); } }; thread.start(); Index: lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java (revision 1542799) +++ lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java (working copy) @@ -10,7 +10,6 @@ import org.apache.lucene.facet.FacetTestCase; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; -import org.apache.lucene.facet.taxonomy.TaxonomyReader.ChildrenIterator; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -219,7 +218,7 @@ return conf; } }; - TaxonomyReader reader = new DirectoryTaxonomyReader(writer); + DirectoryTaxonomyReader reader = new DirectoryTaxonomyReader(writer); int numRounds = random().nextInt(10) + 10; int numCategories = 1; // one for root @@ -229,7 +228,7 @@ writer.addCategory(new CategoryPath(Integer.toString(i), Integer.toString(j))); } numCategories += numCats + 1 /* one for round-parent */; - TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader); + DirectoryTaxonomyReader newtr = TaxonomyReader.openIfChanged(reader); assertNotNull(newtr); reader.close(); reader = newtr; @@ -270,7 +269,7 @@ } }; - TaxonomyReader reader = new DirectoryTaxonomyReader(writer); + DirectoryTaxonomyReader reader = new DirectoryTaxonomyReader(writer); assertEquals(1, reader.getSize()); assertEquals(1, reader.getParallelTaxonomyArrays().parents().length); @@ -280,7 +279,7 @@ iw.forceMerge(1); // now calling openIfChanged should trip on the bug - TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader); + DirectoryTaxonomyReader newtr = TaxonomyReader.openIfChanged(reader); assertNotNull(newtr); reader.close(); reader = newtr; @@ -317,7 +316,7 @@ // a new segment will be created writer.addCategory(new CategoryPath("a")); - TaxonomyReader reader = new DirectoryTaxonomyReader(writer); + DirectoryTaxonomyReader reader = new DirectoryTaxonomyReader(writer); assertEquals(2, reader.getSize()); assertEquals(2, reader.getParallelTaxonomyArrays().parents().length); @@ -325,7 +324,7 @@ iw.forceMerge(1); // now calling openIfChanged should trip on the wrong assert in ParetArray's ctor - TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader); + DirectoryTaxonomyReader newtr = TaxonomyReader.openIfChanged(reader); assertNotNull(newtr); reader.close(); reader = newtr; @@ -492,26 +491,28 @@ DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir); // non existing category - ChildrenIterator it = taxoReader.getChildren(taxoReader.getOrdinal(new CategoryPath("invalid"))); - assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next()); + int[] it = taxoReader.getChildren(taxoReader.getOrdinal(new CategoryPath("invalid"))); + assertSame(TaxonomyReader.NO_CHILDREN, it); // a category with no children it = taxoReader.getChildren(taxoReader.getOrdinal(new CategoryPath("c"))); - assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next()); + assertSame(TaxonomyReader.NO_CHILDREN, it); // arbitrary negative ordinal it = taxoReader.getChildren(-2); - assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next()); + assertSame(TaxonomyReader.NO_CHILDREN, it); // root's children Set roots = new HashSet(Arrays.asList("a", "b", "c")); it = taxoReader.getChildren(TaxonomyReader.ROOT_ORDINAL); + assertNotSame(TaxonomyReader.NO_CHILDREN, it); + int k = 0; while (!roots.isEmpty()) { - CategoryPath root = taxoReader.getPath(it.next()); + CategoryPath root = taxoReader.getPath(it[k++]); assertEquals(1, root.length); assertTrue(roots.remove(root.components[0])); } - assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next()); + assertEquals(TaxonomyReader.INVALID_ORDINAL, it[k]); for (int i = 0; i < 2; i++) { CategoryPath cp = i == 0 ? new CategoryPath("a") : new CategoryPath("b"); @@ -519,7 +520,8 @@ it = taxoReader.getChildren(ordinal); int numChildren = 0; int child; - while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) { + int j = 0; + while ((child = it[j++]) != TaxonomyReader.INVALID_ORDINAL) { CategoryPath path = taxoReader.getPath(child); assertEquals(2, path.length); assertEquals(path.components[0], i == 0 ? "a" : "b");