Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1427505) +++ lucene/CHANGES.txt (working copy) @@ -95,6 +95,14 @@ a setting, you should extend the relevant class. Additionally, FacetSearchParams is now immutable, and requires all FacetRequests to speified at initialization time. (Shai Erera) + +* LUCENE-4647: CategoryDocumentBuilder and EnhancementsDocumentBuilder are replaced + by FacetFields and AssociationsFacetFields respectively. CategoryEnhancement and + AssociationEnhancement were removed in favor of a simplified CategoryAssociation + interface, with CategoryIntAssociation and CategoryFloatAssociation + implementations. + NOTE: indexes that contain category enhancements/associations are not supported + by the new code and should be recreated. (Shai Erera) New Features Index: lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FacetSource.java =================================================================== --- lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FacetSource.java (revision 1427505) +++ lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FacetSource.java (working copy) @@ -19,7 +19,7 @@ import java.io.IOException; -import org.apache.lucene.facet.index.CategoryContainer; +import org.apache.lucene.facet.associations.CategoryAssociationsContainer; /** * Source items for facets. @@ -28,11 +28,13 @@ */ public abstract class FacetSource extends ContentItemsSource { - /** Returns the next {@link CategoryContainer facets content item}. - * Implementations must account for multi-threading, as multiple threads - * can call this method simultaneously. + /** + * Returns the next {@link CategoryAssociationsContainer facets content item}. + * Implementations must account for multi-threading, as multiple threads can + * call this method simultaneously. */ - public abstract CategoryContainer getNextFacets(CategoryContainer facets) throws NoMoreDataException, IOException; + public abstract CategoryAssociationsContainer getNextFacets(CategoryAssociationsContainer facets) + throws NoMoreDataException, IOException; @Override public void resetInputs() throws IOException { Index: lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/RandomFacetSource.java =================================================================== --- lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/RandomFacetSource.java (revision 1427505) +++ lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/RandomFacetSource.java (working copy) @@ -21,7 +21,7 @@ import java.util.Random; import org.apache.lucene.benchmark.byTask.utils.Config; -import org.apache.lucene.facet.index.CategoryContainer; +import org.apache.lucene.facet.associations.CategoryAssociationsContainer; import org.apache.lucene.facet.taxonomy.CategoryPath; /** @@ -45,9 +45,10 @@ private int maxValue = maxDocFacets * maxFacetDepth; @Override - public CategoryContainer getNextFacets(CategoryContainer facets) throws NoMoreDataException, IOException { + public CategoryAssociationsContainer getNextFacets(CategoryAssociationsContainer facets) + throws NoMoreDataException, IOException { if (facets == null) { - facets = new CategoryContainer(); + facets = new CategoryAssociationsContainer(); } else { facets.clear(); } @@ -59,7 +60,7 @@ cp.add(Integer.toString(random.nextInt(maxValue))); addItem(); } - facets.addCategory(cp); + facets.setAssociation(cp, null); addBytes(cp.toString().length()); // very rough approximation } return facets; Index: lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddFacetedDocTask.java =================================================================== --- lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddFacetedDocTask.java (revision 1427505) +++ lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddFacetedDocTask.java (working copy) @@ -19,8 +19,8 @@ import org.apache.lucene.benchmark.byTask.PerfRunData; import org.apache.lucene.benchmark.byTask.feeds.FacetSource; -import org.apache.lucene.facet.index.CategoryContainer; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.associations.CategoryAssociationsContainer; +import org.apache.lucene.facet.index.FacetFields; /** * Add a faceted document. @@ -42,8 +42,8 @@ super(runData); } - private CategoryContainer facets = null; - private CategoryDocumentBuilder categoryDocBuilder = null; + private CategoryAssociationsContainer facets = null; + private FacetFields facetFields = null; private boolean withFacets = true; @Override @@ -53,8 +53,7 @@ facets = getRunData().getFacetSource().getNextFacets(facets); withFacets = getRunData().getConfig().get("with.facets", true); if (withFacets) { - categoryDocBuilder = new CategoryDocumentBuilder(getRunData().getTaxonomyWriter()); - categoryDocBuilder.setCategories(facets); + facetFields = new FacetFields(getRunData().getTaxonomyWriter()); } } @@ -69,7 +68,7 @@ @Override public int doLogic() throws Exception { if (withFacets) { - categoryDocBuilder.build(doc); + facetFields.addFields(doc, facets); } return super.doLogic(); } Index: lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationIndexer.java =================================================================== --- lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationIndexer.java (revision 1427505) +++ lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationIndexer.java (working copy) @@ -3,20 +3,18 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.store.Directory; - -import org.apache.lucene.facet.enhancements.EnhancementsDocumentBuilder; -import org.apache.lucene.facet.enhancements.association.AssociationProperty; +import org.apache.lucene.facet.associations.AssociationsFacetFields; +import org.apache.lucene.facet.associations.CategoryAssociation; +import org.apache.lucene.facet.associations.CategoryAssociationsContainer; import org.apache.lucene.facet.example.ExampleUtils; import org.apache.lucene.facet.example.simple.SimpleUtils; -import org.apache.lucene.facet.index.CategoryContainer; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.store.Directory; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -62,34 +60,31 @@ // create and open a taxonomy writer TaxonomyWriter taxo = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE); + AssociationsFacetFields assocFacetFields = new AssociationsFacetFields(taxo); + // loop over sample documents int nDocsAdded = 0; int nFacetsAdded = 0; for (int docNum = 0; docNum < SimpleUtils.docTexts.length; docNum++) { ExampleUtils.log(" ++++ DOC ID: " + docNum); // obtain the sample categories for current document - CategoryContainer categoryContainer = new CategoryContainer(); + CategoryAssociationsContainer associations = new CategoryAssociationsContainer(); for (CategoryPath path : SimpleUtils.categories[docNum]) { - categoryContainer.addCategory(path); + associations.setAssociation(path, null); ExampleUtils.log("\t ++++ PATH: " + path); + ++nFacetsAdded; } // and also those with associations CategoryPath[] associationsPaths = AssociationUtils.categories[docNum]; - AssociationProperty[] associationProps = AssociationUtils.associations[docNum]; + CategoryAssociation[] associationsValues = AssociationUtils.associations[docNum]; for (int i = 0; i < associationsPaths.length; i++) { - categoryContainer.addCategory(associationsPaths[i], associationProps[i]); + associations.setAssociation(associationsPaths[i], associationsValues[i]); ExampleUtils.log("\t $$$$ Association: (" - + associationsPaths[i] + "," + associationProps[i] + + associationsPaths[i] + "," + associationsValues[i] + ")"); + ++nFacetsAdded; } - // we do not alter indexing parameters! - // a category document builder will add the categories to a document - // once build() is called - CategoryDocumentBuilder categoryDocBuilder = new EnhancementsDocumentBuilder( - taxo, AssociationUtils.assocIndexingParams); - categoryDocBuilder.setCategories(categoryContainer); - // create a plain Lucene document and add some regular Lucene fields // to it Document doc = new Document(); @@ -97,15 +92,13 @@ doc.add(new TextField(SimpleUtils.TEXT, SimpleUtils.docTexts[docNum], Field.Store.NO)); // invoke the category document builder for adding categories to the - // document and, - // as required, to the taxonomy index - categoryDocBuilder.build(doc); + // document and, as required, to the taxonomy index + assocFacetFields.addFields(doc, associations); // finally add the document to the index iw.addDocument(doc); nDocsAdded++; - nFacetsAdded += categoryContainer.size(); } // commit changes. @@ -122,8 +115,7 @@ taxo.close(); iw.close(); - ExampleUtils.log("Indexed " + nDocsAdded + " documents with overall " - + nFacetsAdded + " facets."); + ExampleUtils.log("Indexed " + nDocsAdded + " documents with overall " + nFacetsAdded + " facets."); } } Index: lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationSearcher.java =================================================================== --- lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationSearcher.java (revision 1427505) +++ lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationSearcher.java (working copy) @@ -7,8 +7,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.facet.example.simple.SimpleSearcher; -import org.apache.lucene.facet.search.params.association.AssociationFloatSumFacetRequest; -import org.apache.lucene.facet.search.params.association.AssociationIntSumFacetRequest; +import org.apache.lucene.facet.search.params.associations.AssociationFloatSumFacetRequest; +import org.apache.lucene.facet.search.params.associations.AssociationIntSumFacetRequest; import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; @@ -49,8 +49,7 @@ AssociationIntSumFacetRequest facetRequest = new AssociationIntSumFacetRequest( new CategoryPath("tags"), 10); - List res = SimpleSearcher.searchWithRequest(indexReader, taxo, - AssociationUtils.assocIndexingParams, facetRequest); + List res = SimpleSearcher.searchWithRequest(indexReader, taxo, null, facetRequest); // close readers taxo.close(); @@ -69,8 +68,7 @@ AssociationFloatSumFacetRequest facetRequest = new AssociationFloatSumFacetRequest( new CategoryPath("genre"), 10); - List res = SimpleSearcher.searchWithRequest(indexReader, taxo, - AssociationUtils.assocIndexingParams, facetRequest); + List res = SimpleSearcher.searchWithRequest(indexReader, taxo, null, facetRequest); // close readers taxo.close(); Index: lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationUtils.java =================================================================== --- lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationUtils.java (revision 1427505) +++ lucene/facet/src/examples/org/apache/lucene/facet/example/association/AssociationUtils.java (working copy) @@ -1,10 +1,8 @@ package org.apache.lucene.facet.example.association; -import org.apache.lucene.facet.enhancements.association.AssociationEnhancement; -import org.apache.lucene.facet.enhancements.association.AssociationFloatProperty; -import org.apache.lucene.facet.enhancements.association.AssociationIntProperty; -import org.apache.lucene.facet.enhancements.association.AssociationProperty; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; +import org.apache.lucene.facet.associations.CategoryAssociation; +import org.apache.lucene.facet.associations.CategoryFloatAssociation; +import org.apache.lucene.facet.associations.CategoryIntAssociation; import org.apache.lucene.facet.taxonomy.CategoryPath; /* @@ -47,33 +45,26 @@ } }; - public static AssociationProperty[][] associations = { + public static CategoryAssociation[][] associations = { // Doc #1 associations { /* 3 occurrences for tag 'lucene' */ - new AssociationIntProperty(3), + new CategoryIntAssociation(3), /* 87% confidence level of genre 'computing' */ - new AssociationFloatProperty(0.87f) + new CategoryFloatAssociation(0.87f) }, // Doc #2 associations { /* 1 occurrence for tag 'lucene' */ - new AssociationIntProperty(1), + new CategoryIntAssociation(1), /* 2 occurrences for tag 'solr' */ - new AssociationIntProperty(2), + new CategoryIntAssociation(2), /* 75% confidence level of genre 'computing' */ - new AssociationFloatProperty(0.75f), + new CategoryFloatAssociation(0.75f), /* 34% confidence level of genre 'software' */ - new AssociationFloatProperty(0.34f), + new CategoryFloatAssociation(0.34f), } }; - /** - * Indexing Params: the indexing params to use when dealing with - * associations. - */ - public static final EnhancementsIndexingParams assocIndexingParams = new EnhancementsIndexingParams( - new AssociationEnhancement()); - } Index: lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java =================================================================== --- lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java (revision 1427505) +++ lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java (working copy) @@ -9,21 +9,20 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; - import org.apache.lucene.facet.example.ExampleUtils; import org.apache.lucene.facet.example.simple.SimpleUtils; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.index.params.PerDimensionIndexingParams; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -162,10 +161,8 @@ List facetList = Arrays.asList(cPaths[docNum]); // we do not alter indexing parameters! - // a category document builder will add the categories to a document - // once build() is called - CategoryDocumentBuilder categoryDocBuilder = new CategoryDocumentBuilder( - taxo, iParams).setCategoryPaths(facetList); + // FacetFields adds the categories to the document in addFields() + FacetFields facetFields = new FacetFields(taxo, iParams); // create a plain Lucene document and add some regular Lucene fields // to it @@ -174,7 +171,7 @@ doc.add(new TextField(SimpleUtils.TEXT, docTexts[docNum], Field.Store.NO)); // finally add the document to the index - categoryDocBuilder.build(doc); + facetFields.addFields(doc, facetList); iw.addDocument(doc); nDocsAdded++; Index: lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleIndexer.java =================================================================== --- lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleIndexer.java (revision 1427505) +++ lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleIndexer.java (working copy) @@ -6,17 +6,16 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; +import org.apache.lucene.facet.example.ExampleUtils; +import org.apache.lucene.facet.index.FacetFields; +import org.apache.lucene.facet.taxonomy.CategoryPath; +import org.apache.lucene.facet.taxonomy.TaxonomyWriter; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; -import org.apache.lucene.facet.example.ExampleUtils; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -50,31 +49,29 @@ public static void index (Directory indexDir, Directory taxoDir) throws Exception { // create and open an index writer - IndexWriter iw = new IndexWriter(indexDir, new IndexWriterConfig(ExampleUtils.EXAMPLE_VER, SimpleUtils.analyzer)); + final IndexWriter iw = new IndexWriter(indexDir, + new IndexWriterConfig(ExampleUtils.EXAMPLE_VER, SimpleUtils.analyzer)); // create and open a taxonomy writer - TaxonomyWriter taxo = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE); + final TaxonomyWriter taxo = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE); + // FacetFields adds the categories to the document in addFields() + final FacetFields facetFields = new FacetFields(taxo); + // loop over sample documents int nDocsAdded = 0; int nFacetsAdded = 0; - for (int docNum=0; docNum facetList = Arrays.asList(SimpleUtils.categories[docNum]); - // we do not alter indexing parameters! - // a category document builder will add the categories to a document once build() is called - CategoryDocumentBuilder categoryDocBuilder = new CategoryDocumentBuilder(taxo).setCategoryPaths(facetList); - // create a plain Lucene document and add some regular Lucene fields to it Document doc = new Document(); doc.add(new TextField(SimpleUtils.TITLE, SimpleUtils.docTitles[docNum], Field.Store.YES)); doc.add(new TextField(SimpleUtils.TEXT, SimpleUtils.docTexts[docNum], Field.Store.NO)); - // invoke the category document builder for adding categories to the document and, - // as required, to the taxonomy index - categoryDocBuilder.build(doc); + // add the facet fields to the document + facetFields.addFields(doc, facetList); // finally add the document to the index iw.addDocument(doc); Index: lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsCategoryListBuilder.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsCategoryListBuilder.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsCategoryListBuilder.java (working copy) @@ -0,0 +1,92 @@ +package org.apache.lucene.facet.associations; + +import java.io.IOException; +import java.util.HashMap; + +import org.apache.lucene.facet.index.CategoryListBuilder; +import org.apache.lucene.facet.index.params.CategoryListParams; +import org.apache.lucene.facet.index.params.FacetIndexingParams; +import org.apache.lucene.facet.taxonomy.CategoryPath; +import org.apache.lucene.facet.taxonomy.TaxonomyWriter; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.util.BytesRef; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A {@link CategoryListBuilder} which encodes category-association value pairs + * in addition to regular counting list category ordinals. Every + * category-association pair is written under the respective association's + * {@link CategoryAssociation#getCategoryListID()}. + */ +public class AssociationsCategoryListBuilder extends CategoryListBuilder { + + private final CategoryAssociationsContainer associations; + private final HashMap perAssociationBytes = new HashMap(); + private final ByteArrayDataOutput output = new ByteArrayDataOutput(); + + public AssociationsCategoryListBuilder(CategoryAssociationsContainer associations, + CategoryListParams categoryListParams, FacetIndexingParams indexingParams, TaxonomyWriter taxoWriter) { + super(categoryListParams, indexingParams, taxoWriter); + this.associations = associations; + } + + @Override + public void handle(int ordinal, CategoryPath cp) throws IOException { + super.handle(ordinal, cp); + + // build per-association key BytesRef + CategoryAssociation association = associations.getAssociation(cp); + if (association == null) { + // it is ok to set a null association for a category - it's treated as a + // regular category in that case. + return; + } + + BytesRef bytes = perAssociationBytes.get(association.getCategoryListID()); + if (bytes == null) { + bytes = new BytesRef(); + perAssociationBytes.put(association.getCategoryListID(), bytes); + } + + int maxBytesNeeded = 4 /* int */ + association.maxBytesNeeded(); + if (bytes.bytes.length - bytes.length < maxBytesNeeded) { + bytes.grow(bytes.bytes.length + maxBytesNeeded); + } + + // reset the output to write from bytes.length (current position) until the end + output.reset(bytes.bytes, bytes.length, bytes.bytes.length - bytes.length); + output.writeInt(ordinal); + + // encode the association bytes + association.serialize(output); + + // update BytesRef + bytes.length = output.getPosition(); + } + + @Override + public HashMap finish() { + // build the ordinals list + HashMap result = super.finish(); + // add per association bytes + result.putAll(perAssociationBytes); + return result; + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsCategoryListBuilder.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsDrillDownStream.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsDrillDownStream.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsDrillDownStream.java (working copy) @@ -0,0 +1,74 @@ +package org.apache.lucene.facet.associations; + +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.facet.index.DrillDownStream; +import org.apache.lucene.facet.index.params.FacetIndexingParams; +import org.apache.lucene.facet.taxonomy.CategoryPath; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.util.BytesRef; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A {@link DrillDownStream} which adds to each drill-down token a + * payload according to the {@link CategoryAssociation} defined in the + * {@link CategoryAssociationsContainer}. + * + * @lucene.experimental + */ +public class AssociationsDrillDownStream extends DrillDownStream { + + private final PayloadAttribute payloadAttribute; + private final BytesRef payload; + private final ByteArrayDataOutput output = new ByteArrayDataOutput(); + private final CategoryAssociationsContainer associations; + + public AssociationsDrillDownStream(CategoryAssociationsContainer associations, FacetIndexingParams indexingParams) { + super(associations, indexingParams); + this.associations = associations; + payloadAttribute = addAttribute(PayloadAttribute.class); + BytesRef bytes = payloadAttribute.getPayload(); + if (bytes == null) { + bytes = new BytesRef(new byte[4]); + payloadAttribute.setPayload(bytes); + } + bytes.offset = 0; + this.payload = bytes; + } + + @Override + protected void addAdditionalAttributes(CategoryPath cp, boolean isParent) { + if (isParent) { + return; // associations are not added to parent categories + } + + CategoryAssociation association = associations.getAssociation(cp); + if (association == null) { + // it is ok to set a null association for a category - it's treated as a + // regular category in that case. + return; + } + if (payload.bytes.length < association.maxBytesNeeded()) { + payload.grow(association.maxBytesNeeded()); + } + output.reset(payload.bytes); + association.serialize(output); + payload.length = output.getPosition(); + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsDrillDownStream.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsFacetFields.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsFacetFields.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsFacetFields.java (working copy) @@ -0,0 +1,132 @@ +package org.apache.lucene.facet.associations; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.TextField; +import org.apache.lucene.facet.index.CategoryListBuilder; +import org.apache.lucene.facet.index.DrillDownStream; +import org.apache.lucene.facet.index.FacetFields; +import org.apache.lucene.facet.index.params.CategoryListParams; +import org.apache.lucene.facet.index.params.FacetIndexingParams; +import org.apache.lucene.facet.taxonomy.CategoryPath; +import org.apache.lucene.facet.taxonomy.TaxonomyWriter; +import org.apache.lucene.index.FieldInfo.IndexOptions; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A utility class for adding facet fields to a document. Usually one field will + * be added for all facets, however per the + * {@link FacetIndexingParams#getCategoryListParams(CategoryPath)}, one field + * may be added for every group of facets. + * + * @lucene.experimental + */ +public class AssociationsFacetFields extends FacetFields { + + // The drill-down field is added with a TokenStream, hence why it's based on + // TextField type. However for associations, we store a payload with the + // association value, therefore we set IndexOptions to include positions. + private static final FieldType DRILL_DOWN_TYPE = new FieldType(TextField.TYPE_NOT_STORED); + static { + DRILL_DOWN_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + DRILL_DOWN_TYPE.freeze(); + } + + /** + * Constructs a new instance with the {@link FacetIndexingParams#ALL_PARENTS + * default} facet indexing params. + * + * @param taxonomyWriter + * used to resolve given categories to ordinals + */ + public AssociationsFacetFields(TaxonomyWriter taxonomyWriter) { + super(taxonomyWriter); + } + + /** + * Constructs a new instance with the given facet indexing params. + * + * @param taxonomyWriter + * used to resolve given categories to ordinals + * @param params + * determines under which fields the categories should be indexed + */ + public AssociationsFacetFields(TaxonomyWriter taxonomyWriter, FacetIndexingParams params) { + super(taxonomyWriter, params); + } + + @Override + protected Map> createCategoryListMapping( + Iterable categories) { + CategoryAssociationsContainer categoryAssociations = (CategoryAssociationsContainer) categories; + HashMap> categoryLists = + new HashMap>(); + for (CategoryPath cp : categories) { + // each category may be indexed under a different field, so add it to the right list. + CategoryListParams clp = indexingParams.getCategoryListParams(cp); + CategoryAssociationsContainer clpContainer = (CategoryAssociationsContainer) categoryLists.get(clp); + if (clpContainer == null) { + clpContainer = new CategoryAssociationsContainer(); + categoryLists.put(clp, clpContainer); + } + // DrillDownStream modifies the CategoryPath by calling trim(). That means + // that the source category, as the app ses it, is modified. While for + // most apps this is not a problem, we need to protect against it. If + // CategoryPath will be made immutable, we can stop cloning. + cp = cp.clone(); + clpContainer.setAssociation(cp, categoryAssociations.getAssociation(cp)); + } + return categoryLists; + } + + /** + * Returns a {@link CategoryListBuilder} for encoding the given categories and + * associations. + */ + @Override + protected CategoryListBuilder getCategoryListBuilder(CategoryListParams categoryListParams, + Iterable categories) { + return new AssociationsCategoryListBuilder((CategoryAssociationsContainer) categories, categoryListParams, + indexingParams, taxonomyWriter); + } + + @Override + protected DrillDownStream getDrillDownStream(Iterable categories) { + return new AssociationsDrillDownStream((CategoryAssociationsContainer) categories, indexingParams); + } + + @Override + protected FieldType fieldType() { + return DRILL_DOWN_TYPE; + } + + @Override + public void addFields(Document doc, Iterable categories) throws IOException { + if (!(categories instanceof CategoryAssociationsContainer)) { + throw new IllegalArgumentException("categories must be of type " + + CategoryAssociationsContainer.class.getSimpleName()); + } + super.addFields(doc, categories); + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsFacetFields.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsPayloadIterator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsPayloadIterator.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsPayloadIterator.java (working copy) @@ -0,0 +1,81 @@ +package org.apache.lucene.facet.associations; + +import java.io.IOException; + +import org.apache.lucene.facet.search.PayloadIterator; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.util.BytesRef; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * An iterator over a document's category associations. + * + * @lucene.experimental + */ +public abstract class AssociationsPayloadIterator { + + private final PayloadIterator pi; + private final T association; + + /** + * Marking whether there are associations (at all) in the given index + */ + private boolean hasAssociations = false; + + /** + * Construct a new associations iterator. The given + * {@link CategoryAssociation} is used to deserialize the association values. + * It is assumed that all association values can be deserialized with the + * given {@link CategoryAssociation}. + */ + public AssociationsPayloadIterator(IndexReader reader, String field, T association) throws IOException { + pi = new PayloadIterator(reader, new Term(field, association.getCategoryListID())); + hasAssociations = pi.init(); + this.association = association; + } + + /** + * Skip to the requested document. Returns true iff the document has categort + * association values and they were read successfully. + */ + public boolean setNextDoc(int docId) throws IOException { + if (!hasAssociations) { // there are no associations at all + return false; + } + + if (!pi.setdoc(docId)) { // no associations for the requested document + return false; + } + + BytesRef associations = pi.getPayload(); + ByteArrayDataInput in = new ByteArrayDataInput(associations.bytes, associations.offset, associations.length); + while (!in.eof()) { + int ordinal = in.readInt(); + association.deserialize(in); + handleAssociation(ordinal, association); + } + return true; + } + + /** A hook for extending classes to handle the given association value for the ordinal. */ + protected abstract void handleAssociation(int ordinal, T association); + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/AssociationsPayloadIterator.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryAssociation.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryAssociation.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryAssociation.java (working copy) @@ -0,0 +1,48 @@ +package org.apache.lucene.facet.associations; + +import org.apache.lucene.facet.taxonomy.CategoryPath; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Allows associating an arbitrary value with a {@link CategoryPath}. + * + * @lucene.experimental + */ +public interface CategoryAssociation { + + /** Serializes the associated value into the given {@link DataOutput}. */ + public void serialize(ByteArrayDataOutput output); + + /** Deserializes the association value from the given {@link DataInput}. */ + public void deserialize(ByteArrayDataInput input); + + /** Returns the maximum bytes needed to encode the association value. */ + public int maxBytesNeeded(); + + /** + * Returns the ID of the category association. The ID is used as e.g. the + * term's text under which to encode the association values. + */ + public String getCategoryListID(); + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryAssociation.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryAssociationsContainer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryAssociationsContainer.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryAssociationsContainer.java (working copy) @@ -0,0 +1,59 @@ +package org.apache.lucene.facet.associations; + +import java.util.HashMap; +import java.util.Iterator; + +import org.apache.lucene.facet.taxonomy.CategoryPath; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Holds {@link CategoryAssociation} per {@link CategoryPath}. */ +public class CategoryAssociationsContainer implements Iterable { + + private final HashMap categoryAssociations = + new HashMap(); + + /** + * Adds the {@link CategoryAssociation} for the given {@link CategoryPath + * category}. Overrides any assocation that was previously set. It is ok to + * pass {@code null}, in which case the category will be treated as a regular + * one (i.e. without association value). + */ + public void setAssociation(CategoryPath category, CategoryAssociation association) { + categoryAssociations.put(category, association); + } + + /** + * Returns the {@link CategoryAssociation} that was set for the + * {@link CategoryPath category}, or {@code null} if none was defined. + */ + public CategoryAssociation getAssociation(CategoryPath category) { + return categoryAssociations.get(category); + } + + @Override + public Iterator iterator() { + return categoryAssociations.keySet().iterator(); + } + + /** Clears all category associations. */ + public void clear() { + categoryAssociations.clear(); + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryAssociationsContainer.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryFloatAssociation.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryFloatAssociation.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryFloatAssociation.java (working copy) @@ -0,0 +1,75 @@ +package org.apache.lucene.facet.associations; + +import java.io.IOException; + +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteArrayDataOutput; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** A {@link CategoryAssociation} that associates a float with a category. */ +public class CategoryFloatAssociation implements CategoryAssociation { + + public static final String ASSOCIATION_LIST_ID = "$assoc_float$"; + + private float value; + + public CategoryFloatAssociation() { + // used for deserialization + } + + public CategoryFloatAssociation(float value) { + this.value = value; + } + + @Override + public void serialize(ByteArrayDataOutput output) { + try { + output.writeInt(Float.floatToIntBits(value)); + } catch (IOException e) { + throw new RuntimeException("unexpected exception writing to a byte[]", e); + } + } + + @Override + public void deserialize(ByteArrayDataInput input) { + value = Float.intBitsToFloat(input.readInt()); + } + + @Override + public int maxBytesNeeded() { + // plain integer + return 4; + } + + @Override + public String getCategoryListID() { + return ASSOCIATION_LIST_ID; + } + + /** + * Returns the value associated with a category. If you used + * {@link #CategoryFloatAssociation()}, you should call + * {@link #deserialize(ByteArrayDataInput)} before calling this method, or + * otherwise the value returned is undefined. + */ + public float getValue() { + return value; + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryFloatAssociation.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryIntAssociation.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryIntAssociation.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryIntAssociation.java (working copy) @@ -0,0 +1,75 @@ +package org.apache.lucene.facet.associations; + +import java.io.IOException; + +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteArrayDataOutput; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** A {@link CategoryAssociation} that associates an integer with a category. */ +public class CategoryIntAssociation implements CategoryAssociation { + + public static final String ASSOCIATION_LIST_ID = "$assoc_int$"; + + private int value; + + public CategoryIntAssociation() { + // used for deserialization + } + + public CategoryIntAssociation(int value) { + this.value = value; + } + + @Override + public void serialize(ByteArrayDataOutput output) { + try { + output.writeInt(value); + } catch (IOException e) { + throw new RuntimeException("unexpected exception writing to a byte[]", e); + } + } + + @Override + public void deserialize(ByteArrayDataInput input) { + value = input.readInt(); + } + + @Override + public int maxBytesNeeded() { + // plain integer + return 4; + } + + @Override + public String getCategoryListID() { + return ASSOCIATION_LIST_ID; + } + + /** + * Returns the value associated with a category. If you used + * {@link #CategoryIntAssociation()}, you should call + * {@link #deserialize(ByteArrayDataInput)} before calling this method, or + * otherwise the value returned is undefined. + */ + public int getValue() { + return value; + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/CategoryIntAssociation.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/FloatAssociationsPayloadIterator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/FloatAssociationsPayloadIterator.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/FloatAssociationsPayloadIterator.java (working copy) @@ -0,0 +1,62 @@ +package org.apache.lucene.facet.associations; + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.collections.IntToFloatMap; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * An {@link AssociationsPayloadIterator} over integer association values. + * + * @lucene.experimental + */ +public class FloatAssociationsPayloadIterator extends AssociationsPayloadIterator { + + private final IntToFloatMap ordinalAssociations = new IntToFloatMap(); + + public FloatAssociationsPayloadIterator(IndexReader reader, String field, CategoryFloatAssociation association) + throws IOException { + super(reader, field, association); + } + + @Override + protected void handleAssociation(int ordinal, CategoryFloatAssociation association) { + ordinalAssociations.put(ordinal, association.getValue()); + } + + @Override + public boolean setNextDoc(int docId) throws IOException { + ordinalAssociations.clear(); + return super.setNextDoc(docId); + } + + /** + * Get the float association value for the given ordinal, or + * {@link Float#NaN} in case the ordinal has no association value. + */ + public float getAssociation(int ordinal) { + if (ordinalAssociations.containsKey(ordinal)) { + return ordinalAssociations.get(ordinal); + } + + return Float.NaN; + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/FloatAssociationsPayloadIterator.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java (working copy) @@ -0,0 +1,68 @@ +package org.apache.lucene.facet.associations; + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.collections.IntToIntMap; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * An {@link AssociationsPayloadIterator} over integer association values. + * + * @lucene.experimental + */ +public class IntAssociationsPayloadIterator extends AssociationsPayloadIterator { + + private final IntToIntMap ordinalAssociations = new IntToIntMap(); + + /** + * The long-special-value returned for ordinals which have no associated int + * value. It is not in the int range of values making it a valid mark. + */ + public final static long NO_ASSOCIATION = Integer.MAX_VALUE + 1; + + public IntAssociationsPayloadIterator(IndexReader reader, String field, CategoryIntAssociation association) + throws IOException { + super(reader, field, association); + } + + @Override + protected void handleAssociation(int ordinal, CategoryIntAssociation association) { + ordinalAssociations.put(ordinal, association.getValue()); + } + + @Override + public boolean setNextDoc(int docId) throws IOException { + ordinalAssociations.clear(); + return super.setNextDoc(docId); + } + + /** + * Get the integer association value for the given ordinal, or + * {@link #NO_ASSOCIATION} in case the ordinal has no association value. + */ + public long getAssociation(int ordinal) { + if (ordinalAssociations.containsKey(ordinal)) { + return ordinalAssociations.get(ordinal); + } + + return NO_ASSOCIATION; + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/IntAssociationsPayloadIterator.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/associations/package.html =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/associations/package.html (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/associations/package.html (working copy) @@ -0,0 +1,29 @@ + + + +Category Association + + +

Category Association

+ +Allows association of arbitrary values with a category. The value can be used e.g. to compute +the category's weight during faceted search. Two association implementations exist for +{@link org.apache.lucene.facet.associations.CategoryIntAssociation int} and +{@link org.apache.lucene.facet.associations.CategoryFloatAssociation float} values. + + Property changes on: lucene/facet/src/java/org/apache/lucene/facet/associations/package.html ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/CategoryEnhancement.java (working copy) @@ -1,133 +0,0 @@ -package org.apache.lucene.facet.enhancements; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; -import org.apache.lucene.facet.index.streaming.CategoryParentsStream; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This interface allows easy addition of enhanced category features. Usually, a - * {@link CategoryEnhancement} will correspond to a {@link CategoryProperty}. - *

- * A category enhancement can contribute to the index in two possible ways: - *

    - *
  1. To each category with data relevant to the enhancement, add this data to - * the category's token payload, through - * {@link #getCategoryTokenBytes(CategoryAttribute)}. This data will be read - * during search using {@link #extractCategoryTokenData(byte[], int, int)}.
    2. - *
  2. To each document which contains categories with data relevant to the - * enhancement, add a {@link CategoryListTokenizer} through - * {@link #getCategoryListTokenizer(TokenStream, EnhancementsIndexingParams, TaxonomyWriter)} - * . The {@link CategoryListTokenizer} should add a single token which includes - * all the enhancement relevant data from the categories. The category list - * token's text is defined by {@link #getCategoryListTermText()}.
    3. - *
- * - * @lucene.experimental - */ -public interface CategoryEnhancement { - - /** - * Get the bytes to be added to the category token payload for this - * enhancement. - *

- * NOTE: The returned array is copied, it is recommended to allocate a - * new one each time. - *

- * The bytes generated by this method are the input of - * {@link #extractCategoryTokenData(byte[], int, int)}. - * - * @param categoryAttribute - * The attribute of the category. - * @return The bytes to be added to the category token payload for this - * enhancement. - */ - byte[] getCategoryTokenBytes(CategoryAttribute categoryAttribute); - - /** - * Get the data of this enhancement from a category token payload. - *

- * The input bytes for this method are generated in - * {@link #getCategoryTokenBytes(CategoryAttribute)}. - * - * @param buffer - * The payload buffer. - * @param offset - * The offset of this enhancement's data in the buffer. - * @param length - * The length of this enhancement's data (bytes). - * @return An Object containing the data. - */ - Object extractCategoryTokenData(byte[] buffer, int offset, int length); - - /** - * Declarative method to indicate whether this enhancement generates separate - * category list. - * - * @return {@code true} if generates category list, else {@code false}. - */ - boolean generatesCategoryList(); - - /** - * Returns the text of this enhancement's category list term. - * - * @return The text of this enhancement's category list term. - */ - String getCategoryListTermText(); - - /** - * Get the {@link CategoryListTokenizer} which generates the category list for - * this enhancement. If {@link #generatesCategoryList()} returns {@code false} - * this method will not be called. - * - * @param tokenizer - * The input stream containing categories. - * @param indexingParams - * The indexing params to use. - * @param taxonomyWriter - * The taxonomy to add categories and get their ordinals. - * @return A {@link CategoryListTokenizer} generating the category list for - * this enhancement, with {@code tokenizer} as it's input. - */ - CategoryListTokenizer getCategoryListTokenizer(TokenStream tokenizer, - EnhancementsIndexingParams indexingParams, TaxonomyWriter taxonomyWriter); - - /** - * Get a {@link CategoryProperty} class to be retained when creating - * {@link CategoryParentsStream}. - * - * @return the {@link CategoryProperty} class to be retained when creating - * {@link CategoryParentsStream}, or {@code null} if there is no such - * property. - */ - CategoryProperty getRetainableProperty(); - - /** - * Category enhancements must override {@link Object#equals(Object)}, as it is - * used in - * {@link EnhancementsPayloadIterator#getCategoryData(CategoryEnhancement)}. - */ - @Override - public boolean equals(Object o); -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsCategoryTokenizer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsCategoryTokenizer.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsCategoryTokenizer.java (working copy) @@ -1,121 +0,0 @@ -package org.apache.lucene.facet.enhancements; - -import java.util.List; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.streaming.CategoryTokenizer; -import org.apache.lucene.util.Vint8; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A tokenizer which adds to each category token payload according to the - * {@link CategoryEnhancement}s defined in the given - * {@link EnhancementsIndexingParams}. - * - * @lucene.experimental - */ -public class EnhancementsCategoryTokenizer extends CategoryTokenizer { - - /** - * The data buffer used for payload instance. - */ - protected byte[] payloadBytes; - - /** - * The category enhancements to handle - */ - protected List enhancements; - - /** - * Buffers for enhancement payload bytes - */ - protected byte[][] enhancementBytes; - - private int nStart; - - /** - * Constructor. - * - * @param input - * The stream of category tokens. - * @param indexingParams - * The indexing params to use. - */ - public EnhancementsCategoryTokenizer(TokenStream input, - EnhancementsIndexingParams indexingParams) { - super(input, indexingParams); - payloadBytes = new byte[Vint8.MAXIMUM_BYTES_NEEDED - * (indexingParams.getCategoryEnhancements().size() + 1)]; - enhancements = indexingParams.getCategoryEnhancements(); - if (enhancements != null) { - // create array of bytes per enhancement - enhancementBytes = new byte[enhancements.size()][]; - // write once the number of enhancements in the payload bytes - nStart = Vint8.encode(enhancements.size(), payloadBytes, 0); - } - } - - @Override - protected void setPayload() { - this.payloadAttribute.setPayload(null); - if (enhancements == null) { - return; - } - // clear previous payload content - int nBytes = nStart; - int i = 0; - int nEnhancementBytes = 0; - for (CategoryEnhancement enhancement : enhancements) { - // get payload bytes from each enhancement - enhancementBytes[i] = enhancement - .getCategoryTokenBytes(categoryAttribute); - // write the number of bytes in the payload - if (enhancementBytes[i] == null) { - nBytes += Vint8.encode(0, payloadBytes, nBytes); - } else { - nBytes += Vint8.encode(enhancementBytes[i].length, - payloadBytes, nBytes); - nEnhancementBytes += enhancementBytes[i].length; - } - i++; - } - if (nEnhancementBytes > 0) { - // make sure we have space for all bytes - if (payloadBytes.length < nBytes + nEnhancementBytes) { - byte[] temp = new byte[(nBytes + nEnhancementBytes) * 2]; - System.arraycopy(payloadBytes, 0, temp, 0, nBytes); - payloadBytes = temp; - } - for (i = 0; i < enhancementBytes.length; i++) { - // add the enhancement payload bytes after the existing bytes - if (enhancementBytes[i] != null) { - System.arraycopy(enhancementBytes[i], 0, payloadBytes, - nBytes, enhancementBytes[i].length); - nBytes += enhancementBytes[i].length; - } - } - payload.bytes = payloadBytes; - payload.offset = 0; - payload.length = nBytes; - payloadAttribute.setPayload(payload); - } - } -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsDocumentBuilder.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsDocumentBuilder.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsDocumentBuilder.java (working copy) @@ -1,90 +0,0 @@ -package org.apache.lucene.facet.enhancements; - -import java.io.IOException; -import java.util.List; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.index.streaming.CategoryAttributesStream; -import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; -import org.apache.lucene.facet.index.streaming.CategoryParentsStream; -import org.apache.lucene.facet.index.streaming.CategoryTokenizer; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An {@link EnhancementsDocumentBuilder} is a {@link CategoryDocumentBuilder} - * which adds categories to documents according to the list of - * {@link CategoryEnhancement}s from {@link EnhancementsIndexingParams}. The - * additions over {@link CategoryDocumentBuilder} could be in both category - * tokens, and additional category lists. - * - * @lucene.experimental - */ -public class EnhancementsDocumentBuilder extends CategoryDocumentBuilder { - - /** - * @param params - * Indexing params which include {@link CategoryEnhancement}s. - */ - public EnhancementsDocumentBuilder(TaxonomyWriter taxonomyWriter, - EnhancementsIndexingParams params) { - super(taxonomyWriter, params); - } - - @Override - protected TokenStream getParentsStream(CategoryAttributesStream categoryAttributesStream) { - List toRetainList = ((EnhancementsIndexingParams) indexingParams).getRetainableProperties(); - if (toRetainList != null) { - CategoryParentsStream categoryParentsStream = new CategoryParentsStream( - categoryAttributesStream, taxonomyWriter, indexingParams); - for (CategoryProperty toRetain : toRetainList) { - categoryParentsStream.addRetainableProperty(toRetain.getClass()); - } - return categoryParentsStream; - } - return super.getParentsStream(categoryAttributesStream); - } - - @Override - protected CategoryListTokenizer getCategoryListTokenizer(TokenStream categoryStream) { - CategoryListTokenizer tokenizer = super.getCategoryListTokenizer(categoryStream); - // Add tokenizer for each enhancement that produces category list - for (CategoryEnhancement enhancement : ((EnhancementsIndexingParams) indexingParams) - .getCategoryEnhancements()) { - if (enhancement.generatesCategoryList()) { - tokenizer = enhancement.getCategoryListTokenizer(tokenizer, - (EnhancementsIndexingParams) indexingParams, - taxonomyWriter); - } - } - return tokenizer; - } - - @Override - protected CategoryTokenizer getCategoryTokenizer(TokenStream categoryStream) - throws IOException { - return new EnhancementsCategoryTokenizer(categoryStream, - (EnhancementsIndexingParams) indexingParams); - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsPayloadIterator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsPayloadIterator.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/EnhancementsPayloadIterator.java (working copy) @@ -1,103 +0,0 @@ -package org.apache.lucene.facet.enhancements; - -import java.io.IOException; -import java.util.List; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; - -import org.apache.lucene.facet.search.PayloadIterator; -import org.apache.lucene.util.Vint8; -import org.apache.lucene.util.Vint8.Position; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A {@link PayloadIterator} for iterating over category posting lists generated - * using {@link EnhancementsCategoryTokenizer}. - * - * @lucene.experimental - */ -public class EnhancementsPayloadIterator extends PayloadIterator { - - private CategoryEnhancement[] EnhancedCategories; - int nEnhancements; - private int[] enhancementLength; - private int[] enhancementStart; - - /** - * Constructor. - * - * @param enhancementsList - * A list of the {@link CategoryEnhancement}s from the indexing - * params. - * @param indexReader - * A reader of the index. - * @param term - * The category term to iterate. - * @throws IOException If there is a low-level I/O error. - */ - public EnhancementsPayloadIterator(List enhancementsList, - IndexReader indexReader, Term term) throws IOException { - super(indexReader, term); - EnhancedCategories = enhancementsList.toArray(new CategoryEnhancement[enhancementsList.size()]); - enhancementLength = new int[EnhancedCategories.length]; - enhancementStart = new int[EnhancedCategories.length]; - } - - @Override - public boolean setdoc(int docId) throws IOException { - if (!super.setdoc(docId)) { - return false; - } - - // read header - number of enhancements and their lengths - Position position = new Position(data.offset); - nEnhancements = Vint8.decode(data.bytes, position); - for (int i = 0; i < nEnhancements; i++) { - enhancementLength[i] = Vint8.decode(data.bytes, position); - } - - // set enhancements start points - enhancementStart[0] = position.pos; - for (int i = 1; i < nEnhancements; i++) { - enhancementStart[i] = enhancementStart[i - 1] + enhancementLength[i - 1]; - } - - return true; - } - - /** - * Get the data of the current category and document for a certain - * enhancement, or {@code null} if no such enhancement exists. - * - * @param enhancedCategory - * The category enhancement to apply. - * @return the data of the current category and document for a certain - * enhancement, or {@code null} if no such enhancement exists. - */ - public Object getCategoryData(CategoryEnhancement enhancedCategory) { - for (int i = 0; i < nEnhancements; i++) { - if (enhancedCategory.equals(EnhancedCategories[i])) { - return enhancedCategory.extractCategoryTokenData(data.bytes, - enhancementStart[i], enhancementLength[i]); - } - } - return null; - } -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationEnhancement.java (working copy) @@ -1,172 +0,0 @@ -package org.apache.lucene.facet.enhancements.association; - -import java.util.HashSet; -import java.util.Set; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.enhancements.CategoryEnhancement; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -import org.apache.lucene.util.Vint8; -import org.apache.lucene.util.Vint8.Position; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A {@link CategoryEnhancement} for adding associations data to the index - * (categories with {@link AssociationProperty}s). - * - * @lucene.experimental - */ -public class AssociationEnhancement implements CategoryEnhancement { - - static final String CATEGORY_LIST_TERM_TEXT = "CATEGORY_ASSOCIATION_LIST"; - - /** Property Classes which extend AssociationProperty */ - private static final HashSet> ASSOCIATION_PROPERTY_CLASSES; - - /** Property Classes which do not extend AssociationProperty */ - private static final HashSet> NON_ASSOCIATION_PROPERTY_CLASSES; - - static { - ASSOCIATION_PROPERTY_CLASSES = new HashSet>(); - NON_ASSOCIATION_PROPERTY_CLASSES = new HashSet>(); - } - - /** - * For a given class which extends a CategoryProperty, answers whether it is - * an instance of AssociationProperty (AP) or not.
- * This method is a cheaper replacement for a call to - * instanceof. It has two HashSets - one for classes which are - * an extension to AP and one for the classes which are not. Whenever a - * property class is introduced: - *

    - *
  • if it is known as a property class extending AP (contained in the - * validHashSet)- returns true
    • - *
  • if it is known as a property class NOT extending AP - returns false
    • - *
  • - * If it was not matched against both sets, it calls 'instanceof' to find - * out if it extends AP, puts it in the matching Set and returning true or - * false accordingly
    • - *
- * - * NOTE: 'instanceof' is only called once per a Class (not instance) of a - * property. And as there are few properties (currently 4 concrete - * implementations) the two sets would be rather small - */ - public static boolean isAssociationProperty(Class clazz) { - if (ASSOCIATION_PROPERTY_CLASSES.contains(clazz)) { - return true; - } - - if (NON_ASSOCIATION_PROPERTY_CLASSES.contains(clazz)) { - return false; - } - - if (AssociationProperty.class.isAssignableFrom(clazz)) { - ASSOCIATION_PROPERTY_CLASSES.add(clazz); - return true; - } - - NON_ASSOCIATION_PROPERTY_CLASSES.add(clazz); - return false; - } - - @Override - public boolean generatesCategoryList() { - return true; - } - - @Override - public String getCategoryListTermText() { - return CATEGORY_LIST_TERM_TEXT; - } - - @Override - public CategoryListTokenizer getCategoryListTokenizer( - TokenStream tokenizer, EnhancementsIndexingParams indexingParams, - TaxonomyWriter taxonomyWriter) { - return new AssociationListTokenizer(tokenizer, indexingParams, this); - } - - @Override - public byte[] getCategoryTokenBytes(CategoryAttribute categoryAttribute) { - - AssociationProperty property = getAssociationProperty(categoryAttribute); - - if (property == null) { - return null; - } - - int association = property.getAssociation(); - int bytesNeeded = Vint8.bytesNeeded(association); - byte[] buffer = new byte[bytesNeeded]; - Vint8.encode(association, buffer, 0); - return buffer; - } - - public static AssociationProperty getAssociationProperty( - CategoryAttribute categoryAttribute) { - AssociationProperty property = null; - Set> propertyClasses = categoryAttribute - .getPropertyClasses(); - if (propertyClasses == null) { - return null; - } - for (Class clazz : propertyClasses) { - if (isAssociationProperty(clazz)) { - property = (AssociationProperty) categoryAttribute - .getProperty(clazz); - break; - } - } - return property; - } - - @Override - public Object extractCategoryTokenData(byte[] buffer, int offset, int length) { - if (length == 0) { - return null; - } - Integer i = Integer.valueOf(Vint8.decode(buffer, new Position(offset))); - return i; - } - - @Override - public CategoryProperty getRetainableProperty() { - return null; - } - - @Override - public boolean equals(Object o) { - if (o == this) { - return true; - } - return (o instanceof AssociationEnhancement); - } - - @Override - public int hashCode() { - return super.hashCode(); - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationFloatProperty.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationFloatProperty.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationFloatProperty.java (working copy) @@ -1,75 +0,0 @@ -package org.apache.lucene.facet.enhancements.association; - -import org.apache.lucene.facet.index.attributes.CategoryProperty; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An {@link AssociationProperty} which treats the association as float - the - * association bits are actually float bits, and thus merging two associations - * is done by float summation. - * - * @lucene.experimental - */ -public class AssociationFloatProperty extends AssociationProperty { - - /** - * Constructor. - * - * @param value - * The association value. - */ - public AssociationFloatProperty(float value) { - super(Float.floatToIntBits(value)); - } - - @Override - public boolean equals(Object other) { - if (other == this) { - return true; - } - if (!(other instanceof AssociationFloatProperty)) { - return false; - } - AssociationFloatProperty o = (AssociationFloatProperty) other; - return o.association == this.association; - } - - @Override - public int hashCode() { - return "AssociationFloatProperty".hashCode() * 31 + (int) association; - } - - @Override - public void merge(CategoryProperty other) { - AssociationFloatProperty o = (AssociationFloatProperty) other; - this.association = Float.floatToIntBits(Float - .intBitsToFloat((int) this.association) - + Float.intBitsToFloat((int) o.association)); - } - - public float getFloatAssociation() { - return Float.intBitsToFloat((int) association); - } - - @Override - public String toString() { - return getClass().getSimpleName() + ": " + Float.intBitsToFloat(getAssociation()); - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationIntProperty.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationIntProperty.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationIntProperty.java (working copy) @@ -1,61 +0,0 @@ -package org.apache.lucene.facet.enhancements.association; - -import org.apache.lucene.facet.index.attributes.CategoryProperty; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An {@link AssociationProperty} which treats the association as int - merges - * two associations by summation. - * - * @lucene.experimental - */ -public class AssociationIntProperty extends AssociationProperty { - - /** - * @param value - * The association value. - */ - public AssociationIntProperty(int value) { - super(value); - } - - @Override - public boolean equals(Object other) { - if (other == this) { - return true; - } - if (!(other instanceof AssociationIntProperty)) { - return false; - } - AssociationIntProperty o = (AssociationIntProperty) other; - return o.association == this.association; - } - - @Override - public int hashCode() { - return "AssociationIntProperty".hashCode() * 31 + (int) association; - } - - @Override - public void merge(CategoryProperty other) { - AssociationIntProperty o = (AssociationIntProperty) other; - this.association += o.association; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationListTokenizer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationListTokenizer.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationListTokenizer.java (working copy) @@ -1,92 +0,0 @@ -package org.apache.lucene.facet.enhancements.association; - -import java.io.IOException; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.enhancements.CategoryEnhancement; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.CategoryListPayloadStream; -import org.apache.lucene.facet.index.attributes.OrdinalProperty; -import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; -import org.apache.lucene.util.encoding.SimpleIntEncoder; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Tokenizer for associations of a category - * - * @lucene.experimental - */ -public class AssociationListTokenizer extends CategoryListTokenizer { - - protected CategoryListPayloadStream payloadStream; - - private String categoryListTermText; - - public AssociationListTokenizer(TokenStream input, - EnhancementsIndexingParams indexingParams, CategoryEnhancement enhancement) { - super(input, indexingParams); - categoryListTermText = enhancement.getCategoryListTermText(); - } - - @Override - protected void handleStartOfInput() throws IOException { - payloadStream = null; - } - - @Override - public final boolean incrementToken() throws IOException { - if (input.incrementToken()) { - if (categoryAttribute != null) { - AssociationProperty associationProperty = AssociationEnhancement - .getAssociationProperty(categoryAttribute); - if (associationProperty != null - && associationProperty.hasBeenSet()) { - OrdinalProperty ordinalProperty = (OrdinalProperty) categoryAttribute - .getProperty(OrdinalProperty.class); - if (ordinalProperty == null) { - throw new IOException( - "Error: Association without ordinal"); - } - - if (payloadStream == null) { - payloadStream = new CategoryListPayloadStream( - new SimpleIntEncoder()); - } - payloadStream.appendIntToStream(ordinalProperty - .getOrdinal()); - payloadStream.appendIntToStream(associationProperty - .getAssociation()); - } - } - return true; - } - if (payloadStream != null) { - termAttribute.setEmpty().append(categoryListTermText); - payload.bytes = payloadStream.convertStreamToByteArray(); - payload.offset = 0; - payload.length = payload.bytes.length; - payloadAttribute.setPayload(payload); - payloadStream = null; - return true; - } - return false; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationProperty.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationProperty.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationProperty.java (working copy) @@ -1,73 +0,0 @@ -package org.apache.lucene.facet.enhancements.association; - -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryProperty; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A {@link CategoryProperty} associating a single integer value to a - * {@link CategoryAttribute}. It should be used to describe the association - * between the category and the document. - *

- * This class leave to extending classes the definition of - * {@link #merge(CategoryProperty)} policy for the integer associations. - *

- * Note: The association value is added both to a special category list, - * and to the category tokens. - * - * @see AssociationEnhancement - * @lucene.experimental - */ -public abstract class AssociationProperty implements CategoryProperty { - - protected long association = Integer.MAX_VALUE + 1; - - /** - * Construct an {@link AssociationProperty}. - * - * @param value - * The association value. - */ - public AssociationProperty(int value) { - this.association = value; - } - - /** - * Returns the association value. - * - * @return The association value. - */ - public int getAssociation() { - return (int) association; - } - - /** - * Returns whether this attribute has been set (not all categories have an - * association). - */ - public boolean hasBeenSet() { - return this.association <= Integer.MAX_VALUE; - } - - @Override - public String toString() { - return getClass().getSimpleName() + ": " + association; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationsPayloadIterator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationsPayloadIterator.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/AssociationsPayloadIterator.java (working copy) @@ -1,235 +0,0 @@ -package org.apache.lucene.facet.enhancements.association; - -import java.io.IOException; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; - -import org.apache.lucene.facet.index.params.CategoryListParams; -import org.apache.lucene.facet.search.PayloadIntDecodingIterator; -import org.apache.lucene.util.collections.IntIterator; -import org.apache.lucene.util.collections.IntToIntMap; -import org.apache.lucene.util.encoding.SimpleIntDecoder; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Allows easy iteration over the associations payload, decoding and breaking it - * to (ordinal, value) pairs, stored in a hash. - * - * @lucene.experimental - */ -public class AssociationsPayloadIterator { - - /** - * Default Term for associations - */ - public static final Term ASSOCIATION_POSTING_TERM = new Term( - CategoryListParams.DEFAULT_TERM.field(), - AssociationEnhancement.CATEGORY_LIST_TERM_TEXT); - - /** - * Hash mapping to ordinals to the associated int value - */ - private IntToIntMap ordinalToAssociationMap; - - /** - * An inner payload decoder which actually goes through the posting and - * decode the ints representing the ordinals and the values - */ - private PayloadIntDecodingIterator associationPayloadIter; - - /** - * Marking whether there are associations (at all) in the given index - */ - private boolean hasAssociations = false; - - /** - * The long-special-value returned for ordinals which have no associated int - * value. It is not in the int range of values making it a valid mark. - */ - public final static long NO_ASSOCIATION = Integer.MAX_VALUE + 1; - - /** - * Construct a new association-iterator, initializing the inner payload - * iterator, with the supplied term and checking whether there are any - * associations within the given index - * - * @param reader - * a reader containing the postings to be iterated - * @param field - * the field containing the relevant associations list term - */ - public AssociationsPayloadIterator(IndexReader reader, String field) - throws IOException { - // Initialize the payloadDecodingIterator - associationPayloadIter = new PayloadIntDecodingIterator( - reader, - // TODO (Facet): should consolidate with AssociationListTokenizer which - // uses AssociationEnhancement.getCatTermText() - new Term(field, AssociationEnhancement.CATEGORY_LIST_TERM_TEXT), - new SimpleIntDecoder()); - - // Check whether there are any associations - hasAssociations = associationPayloadIter.init(); - - ordinalToAssociationMap = new IntToIntMap(); - } - - /** - * Skipping to the next document, fetching its associations & populating the - * map. - * - * @param docId - * document id to be skipped to - * @return true if the document contains associations and they were fetched - * correctly. false otherwise. - * @throws IOException - * on error - */ - public boolean setNextDoc(int docId) throws IOException { - ordinalToAssociationMap.clear(); - boolean docContainsAssociations = false; - try { - docContainsAssociations = fetchAssociations(docId); - } catch (IOException e) { - IOException ioe = new IOException( - "An Error occured while reading a document's associations payload (docId=" - + docId + ")"); - ioe.initCause(e); - throw ioe; - } - - return docContainsAssociations; - } - - /** - * Get int association value for the given ordinal.
- * The return is either an int value casted as long if the ordinal has an - * associated value. Otherwise the returned value would be - * {@link #NO_ASSOCIATION} which is 'pure long' value (e.g not in the int - * range of values) - * - * @param ordinal - * for which the association value is requested - * @return the associated int value (encapsulated in a long) if the ordinal - * had an associated value, or {@link #NO_ASSOCIATION} otherwise - */ - public long getAssociation(int ordinal) { - if (ordinalToAssociationMap.containsKey(ordinal)) { - return ordinalToAssociationMap.get(ordinal); - } - - return NO_ASSOCIATION; - } - - /** - * Get an iterator over the ordinals which has an association for the - * document set by {@link #setNextDoc(int)}. - */ - public IntIterator getAssociatedOrdinals() { - return ordinalToAssociationMap.keyIterator(); - } - - /** - * Skips to the given docId, getting the values in pairs of (ordinal, value) - * and populating the map - * - * @param docId - * document id owning the associations - * @return true if associations were fetched successfully, false otherwise - * @throws IOException - * on error - */ - private boolean fetchAssociations(int docId) throws IOException { - // No associations at all? don't bother trying to seek the docID in the - // posting - if (!hasAssociations) { - return false; - } - - // No associations for this document? well, nothing to decode than, - // return false - if (!associationPayloadIter.skipTo(docId)) { - return false; - } - - // loop over all the values decoded from the payload in pairs. - for (;;) { - // Get the ordinal - long ordinal = associationPayloadIter.nextCategory(); - - // if no ordinal - it's the end of data, break the loop - if (ordinal > Integer.MAX_VALUE) { - break; - } - - // get the associated value - long association = associationPayloadIter.nextCategory(); - // If we're at this step - it means we have an ordinal, do we have - // an association for it? - if (association > Integer.MAX_VALUE) { - // No association!!! A Broken Pair!! PANIC! - throw new IOException( - "ERROR! Associations should come in pairs of (ordinal, value), yet this payload has an odd number of values! (docId=" - + docId + ")"); - } - // Populate the map with the given ordinal and association pair - ordinalToAssociationMap.put((int) ordinal, (int) association); - } - - return true; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime - * result - + ((associationPayloadIter == null) ? 0 - : associationPayloadIter.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - - if (obj == null) { - return false; - } - - if (getClass() != obj.getClass()) { - return false; - } - - AssociationsPayloadIterator other = (AssociationsPayloadIterator) obj; - if (associationPayloadIter == null) { - if (other.associationPayloadIter != null) { - return false; - } - } else if (!associationPayloadIter.equals(other.associationPayloadIter)) { - return false; - } - return true; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/package.html =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/package.html (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/association/package.html (working copy) @@ -1,29 +0,0 @@ - - - -Association category enhancements - - -

Association category enhancements

- -A {@link org.apache.lucene.facet.enhancements.CategoryEnhancement CategoryEnhancement} -for adding associations data to the index (categories with -{@link org.apache.lucene.facet.enhancements.association.AssociationProperty AssociationProperty}'s). - - - \ No newline at end of file Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/package.html =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/package.html (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/package.html (working copy) @@ -1,48 +0,0 @@ - - - -Enhanced category features - - -

Enhanced category features

- -Mechanisms for addition of enhanced category features. -

A {@link org.apache.lucene.facet.enhancements.CategoryEnhancement CategoryEnhancement} -(which can correspond to a -{@link org.apache.lucene.facet.index.attributes.CategoryProperty CategoryProperty}) -can contribute to the index in two possible ways: -

    -
  1. To each category with data relevant to the enhancement, - add this data to the category's token payload, through - {@link org.apache.lucene.facet.enhancements.CategoryEnhancement#getCategoryTokenBytes(CategoryAttribute) CategoryEnhancement.getCategoryTokenBytes()}. - This data will be read during search using - {@link org.apache.lucene.facet.enhancements.CategoryEnhancement#extractCategoryTokenData(byte[], int, int) CategoryEnhancement.extractCategoryTokenData()}. -
    2. -
  2. To each document which contains categories with data relevant to the enhancement, add a - {@link org.apache.lucene.facet.index.streaming.CategoryListTokenizer CategoryListTokenizer} through - {@link org.apache.lucene.facet.enhancements.CategoryEnhancement#getCategoryListTokenizer CategoryEnhancement.getCategoryListTokenizer()} . - The - {@link org.apache.lucene.facet.index.streaming.CategoryListTokenizer CategoryListTokenizer} - should add a single token which includes all the enhancement relevant data from the categories. - The category list token's text is defined by - {@link org.apache.lucene.facet.enhancements.CategoryEnhancement#getCategoryListTermText() CategoryEnhancement.getCategoryListTermText()}. -
    3. -
- - - Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParams.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParams.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParams.java (working copy) @@ -1,107 +0,0 @@ -package org.apache.lucene.facet.enhancements.params; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.apache.lucene.facet.enhancements.CategoryEnhancement; -import org.apache.lucene.facet.enhancements.EnhancementsDocumentBuilder; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.index.params.CategoryListParams; -import org.apache.lucene.facet.index.params.PerDimensionIndexingParams; -import org.apache.lucene.facet.index.streaming.CategoryParentsStream; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A {@link PerDimensionIndexingParams} for defining {@link CategoryEnhancement - * category enhancements}. Must contain at least one enhancement, and when there - * are more than one, their order matters. - * - * @see #getCategoryEnhancements() - * @see EnhancementsDocumentBuilder - - * @lucene.experimental - */ -public class EnhancementsIndexingParams extends PerDimensionIndexingParams { - - private final List enhancements; - - /** - * Initializes with the given enhancements - * - * @throws IllegalArgumentException - * if no enhancements are provided - */ - public EnhancementsIndexingParams(CategoryEnhancement... enhancements) { - this(DEFAULT_CATEGORY_LIST_PARAMS, Collections. emptyMap(), enhancements); - } - - /** - * Initializes with the given enhancements and category list params mappings. - * - * @see PerDimensionIndexingParams#PerDimensionIndexingParams(Map, CategoryListParams) - * @throws IllegalArgumentException - * if no enhancements are provided - */ - public EnhancementsIndexingParams(CategoryListParams categoryListParams, - Map paramsMap, CategoryEnhancement... enhancements) { - super(paramsMap, categoryListParams); - validateparams(enhancements); - this.enhancements = Arrays.asList(enhancements); - } - - private void validateparams(CategoryEnhancement... enhancements) { - if (enhancements == null || enhancements.length < 1) { - throw new IllegalArgumentException("at least one enhancement is required"); - } - } - - /** - * Returns the list of {@link CategoryEnhancement} as were given at - * intialization time. You are not expected to modify the list. The order of - * the enhancements dictates the order in which they are written in the - * document. - */ - public List getCategoryEnhancements() { - return enhancements; - } - - /** - * Returns a list of {@link CategoryProperty} which should be retained when - * creating {@link CategoryParentsStream}, or {@code null} if there are no - * such properties. - */ - public List getRetainableProperties() { - List props = new ArrayList(); - for (CategoryEnhancement enhancement : enhancements) { - CategoryProperty prop = enhancement.getRetainableProperty(); - if (prop != null) { - props.add(prop); - } - } - if (props.isEmpty()) { - return null; - } - return props; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/package.html =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/package.html (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/enhancements/params/package.html (working copy) @@ -1,32 +0,0 @@ - - - -Enhanced category features - - -

Enhanced category features

- -{@link org.apache.lucene.facet.index.params.FacetIndexingParams FacetIndexingParams} -used by -{@link org.apache.lucene.facet.enhancements.EnhancementsDocumentBuilder EnhancementsDocumentBuilder} -for adding -{@link org.apache.lucene.facet.enhancements.CategoryEnhancement CategoryEnhancement}'s -to the indexing parameters, and accessing them during indexing and search. - - - \ No newline at end of file Index: lucene/facet/src/java/org/apache/lucene/facet/index/CategoryContainer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/CategoryContainer.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/CategoryContainer.java (working copy) @@ -1,269 +0,0 @@ -package org.apache.lucene.facet.index; - -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -import org.apache.lucene.util.Attribute; - -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryAttributeImpl; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A container to add categories which are to be introduced to - * {@link CategoryDocumentBuilder#setCategories(Iterable)}. Categories can be - * added with Properties. - * - * @lucene.experimental - */ -public class CategoryContainer implements Iterable, Serializable { - - protected transient Map map; - - /** - * Constructor. - */ - public CategoryContainer() { - map = new HashMap(); - } - - /** - * Add a category. - * - * @param categoryPath - * The path of the category. - * @return The {@link CategoryAttribute} of the category. - */ - public CategoryAttribute addCategory(CategoryPath categoryPath) { - return mapCategoryAttribute(categoryPath); - } - - /** - * Add a category with a property. - * - * @param categoryPath - * The path of the category. - * @param property - * The property to associate to the category. - * @return The {@link CategoryAttribute} of the category. - */ - public CategoryAttribute addCategory(CategoryPath categoryPath, - CategoryProperty property) { - /* - * This method is a special case of addCategory with multiple - * properties, but it is kept here for two reasons: 1) Using the array - * version has some performance cost, and 2) it is expected that most - * calls will be for this version (single property). - */ - CategoryAttribute ca = mapCategoryAttribute(categoryPath); - ca.addProperty(property); - return ca; - } - - /** - * Add a category with multiple properties. - * - * @param categoryPath - * The path of the category. - * @param properties - * The properties to associate to the category. - * @return The {@link CategoryAttribute} of the category. - */ - public CategoryAttribute addCategory(CategoryPath categoryPath, - CategoryProperty... properties) { - CategoryAttribute ca = mapCategoryAttribute(categoryPath); - for (CategoryProperty attribute : properties) { - ca.addProperty(attribute); - } - return ca; - } - - /** - * Add an entire {@link CategoryAttribute}. - * - * @param categoryAttribute - * The {@link CategoryAttribute} to add. - * @return The {@link CategoryAttribute} of the category (could be different - * from the one provided). - */ - public CategoryAttribute addCategory(CategoryAttribute categoryAttribute) { - CategoryAttribute ca = mapCategoryAttribute(categoryAttribute - .getCategoryPath()); - Set> propertyClasses = categoryAttribute - .getPropertyClasses(); - if (propertyClasses != null) { - for (Class propertyClass : propertyClasses) { - ca.addProperty(categoryAttribute.getProperty(propertyClass)); - } - } - return ca; - } - - /** - * Get the {@link CategoryAttribute} object for a specific - * {@link CategoryPath}, from the map. - */ - private final CategoryAttribute mapCategoryAttribute( - CategoryPath categoryPath) { - CategoryAttribute ca = map.get(categoryPath); - if (ca == null) { - ca = new CategoryAttributeImpl(categoryPath); - map.put(categoryPath, ca); - } - return ca; - } - - /** - * Get the {@link CategoryAttribute} this container has for a certain - * category, or {@code null} if the category is not in the container. - * - * @param categoryPath - * The category path of the requested category. - */ - public CategoryAttribute getCategoryAttribute(CategoryPath categoryPath) { - return map.get(categoryPath); - } - - @Override - public Iterator iterator() { - return map.values().iterator(); - } - - /** - * Remove all categories. - */ - public void clear() { - map.clear(); - } - - /** Add the categories from another {@link CategoryContainer} to this one. */ - public void merge(CategoryContainer other) { - for (CategoryAttribute categoryAttribute : other.map.values()) { - addCategory(categoryAttribute); - } - } - - /** - * Get the number of categories in the container. - * - * @return The number of categories in the container. - */ - public int size() { - return map.size(); - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder("CategoryContainer"); - for (CategoryAttribute ca : map.values()) { - builder.append('\n'); - builder.append('\t'); - builder.append(ca.toString()); - } - return builder.toString(); - } - - /** - * Serialize object content to given {@link ObjectOutputStream} - */ - private void writeObject(ObjectOutputStream out) throws IOException { - out.defaultWriteObject(); - // write the number of categories - out.writeInt(size()); - // write the category attributes - for (CategoryAttribute ca : this) { - serializeCategoryAttribute(out, ca); - } - } - - /** - * Serialize each of the {@link CategoryAttribute}s to the given - * {@link ObjectOutputStream}.
- * NOTE: {@link CategoryProperty}s are {@link Serializable}, but do not - * assume that Lucene's {@link Attribute}s are as well - * @throws IOException If there is a low-level I/O error. - */ - protected void serializeCategoryAttribute(ObjectOutputStream out, - CategoryAttribute ca) throws IOException { - out.writeObject(ca.getCategoryPath()); - Set> propertyClasses = ca.getPropertyClasses(); - if (propertyClasses != null) { - out.writeInt(propertyClasses.size()); - for (Class clazz : propertyClasses) { - out.writeObject(ca.getProperty(clazz)); - } - } else { - out.writeInt(0); - } - } - - /** - * Deserialize object from given {@link ObjectInputStream} - */ - private void readObject(ObjectInputStream in) throws IOException, - ClassNotFoundException { - in.defaultReadObject(); - map = new HashMap(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { - deserializeCategoryAttribute(in); - } - } - - /** - * De-Serialize each of the {@link CategoryAttribute}s from the given - * {@link ObjectInputStream}. - */ - protected void deserializeCategoryAttribute(ObjectInputStream in) - throws IOException, ClassNotFoundException { - CategoryPath cp = (CategoryPath) in.readObject(); - int nProperties = in.readInt(); - if (nProperties == 0) { - addCategory(cp); - } else { - for (int j = 0; j < nProperties; j++) { - CategoryProperty property = (CategoryProperty) in.readObject(); - addCategory(cp, property); - } - } - } - - @Override - public boolean equals(Object o) { - if (! (o instanceof CategoryContainer)) { - return false; - } - - CategoryContainer that = (CategoryContainer)o; - return this.map.equals(that.map); - } - - @Override - public int hashCode() { - return map.hashCode(); - } -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/CategoryDocumentBuilder.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/CategoryDocumentBuilder.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/CategoryDocumentBuilder.java (working copy) @@ -1,291 +0,0 @@ -package org.apache.lucene.facet.index; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.TextField; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryAttributesIterable; -import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy; -import org.apache.lucene.facet.index.categorypolicy.PathPolicy; -import org.apache.lucene.facet.index.params.FacetIndexingParams; -import org.apache.lucene.facet.index.streaming.CategoryAttributesStream; -import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; -import org.apache.lucene.facet.index.streaming.CategoryParentsStream; -import org.apache.lucene.facet.index.streaming.CategoryTokenizer; -import org.apache.lucene.facet.index.streaming.CountingListTokenizer; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A utility class which allows attachment of {@link CategoryPath}s or - * {@link CategoryAttribute}s to a given document using a taxonomy.
- * Construction could be done with either a given {@link FacetIndexingParams} or - * the default implementation {@link FacetIndexingParams}.
- * A CategoryDocumentBuilder can be reused by repeatedly setting the categories - * and building the document. Categories are provided either as - * {@link CategoryAttribute} elements through {@link #setCategories(Iterable)}, - * or as {@link CategoryPath} elements through - * {@link #setCategoryPaths(Iterable)}. - *

- * Note that both {@link #setCategories(Iterable)} and - * {@link #setCategoryPaths(Iterable)} return this - * {@link CategoryDocumentBuilder}, allowing the following pattern: {@code new - * CategoryDocumentBuilder(taxonomy, - * params).setCategories(categories).build(doc)}. - * - * @lucene.experimental - */ -public class CategoryDocumentBuilder { - - /** - * A {@link TaxonomyWriter} for adding categories and retrieving their - * ordinals. - */ - protected final TaxonomyWriter taxonomyWriter; - - /** - * Parameters to be used when indexing categories. - */ - protected final FacetIndexingParams indexingParams; - - /** - * A list of fields which is filled at ancestors' construction and used - * during {@link CategoryDocumentBuilder#build(Document)}. - */ - protected final ArrayList fieldList = new ArrayList(); - - protected Map> categoriesMap; - - /** - * Creating a facets document builder with default facet indexing parameters. - * - * @param taxonomyWriter - * to which new categories will be added, as well as translating - * known categories to ordinals - * - * @see #CategoryDocumentBuilder(TaxonomyWriter, FacetIndexingParams) - */ - public CategoryDocumentBuilder(TaxonomyWriter taxonomyWriter) { - this(taxonomyWriter, FacetIndexingParams.ALL_PARENTS); - } - - /** - * Creating a facets document builder with a given facet indexing parameters - * object.
- * - * @param taxonomyWriter - * to which new categories will be added, as well as translating - * known categories to ordinals - * @param params - * holds all parameters the indexing process should use such as - * category-list parameters - */ - public CategoryDocumentBuilder(TaxonomyWriter taxonomyWriter, - FacetIndexingParams params) { - this.taxonomyWriter = taxonomyWriter; - this.indexingParams = params; - this.categoriesMap = new HashMap>(); - } - - /** - * Set the categories of the document builder from an {@link Iterable} of - * {@link CategoryPath} objects. - * - * @param categoryPaths - * An iterable of CategoryPath objects which holds the categories - * (facets) which will be added to the document at - * {@link #build(Document)} - * @return This CategoryDocumentBuilder, to enable this one line call: - * {@code new} {@link #CategoryDocumentBuilder(TaxonomyWriter)}. - * {@link #setCategoryPaths(Iterable)}.{@link #build(Document)}. - * @throws IOException If there is a low-level I/O error. - */ - public CategoryDocumentBuilder setCategoryPaths( - Iterable categoryPaths) throws IOException { - if (categoryPaths == null) { - fieldList.clear(); - return this; - } - return setCategories(new CategoryAttributesIterable(categoryPaths)); - } - - /** - * Set the categories of the document builder from an {@link Iterable} of - * {@link CategoryAttribute} objects. - * - * @param categories - * An iterable of {@link CategoryAttribute} objects which holds - * the categories (facets) which will be added to the document at - * {@link #build(Document)} - * @return This CategoryDocumentBuilder, to enable this one line call: - * {@code new} {@link #CategoryDocumentBuilder(TaxonomyWriter)}. - * {@link #setCategories(Iterable)}.{@link #build(Document)}. - * @throws IOException If there is a low-level I/O error. - */ - public CategoryDocumentBuilder setCategories( - Iterable categories) throws IOException { - fieldList.clear(); - if (categories == null) { - return this; - } - - // get field-name to a list of facets mapping as different facets could - // be added to different category-lists on different fields - fillCategoriesMap(categories); - - // creates a different stream for each different field - for (Entry> e : categoriesMap - .entrySet()) { - // create a category attributes stream for the array of facets - CategoryAttributesStream categoryAttributesStream = new CategoryAttributesStream( - e.getValue()); - - // Set a suitable {@link TokenStream} using - // CategoryParentsStream, followed by CategoryListTokenizer and - // CategoryTokenizer composition (the ordering of the last two is - // not mandatory). - CategoryParentsStream parentsStream = (CategoryParentsStream) getParentsStream(categoryAttributesStream); - CategoryListTokenizer categoryListTokenizer = getCategoryListTokenizer(parentsStream); - CategoryTokenizer stream = getCategoryTokenizer(categoryListTokenizer); - - // Finally creating a suitable field with stream and adding it to a - // master field-list, used during the build process (see - // super.build()) - FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); - ft.setOmitNorms(true); - fieldList.add(new Field(e.getKey(), stream, ft)); - } - - return this; - } - - /** - * Get a stream of categories which includes the parents, according to - * policies defined in indexing parameters. - * - * @param categoryAttributesStream - * The input stream - * @return The parents stream. - * @see OrdinalPolicy OrdinalPolicy (for policy of adding category tokens for parents) - * @see PathPolicy PathPolicy (for policy of adding category list tokens for parents) - */ - protected TokenStream getParentsStream( - CategoryAttributesStream categoryAttributesStream) { - return new CategoryParentsStream(categoryAttributesStream, - taxonomyWriter, indexingParams); - } - - /** - * Fills the categories mapping between a field name and a list of - * categories that belongs to it according to this builder's - * {@link FacetIndexingParams} object - * - * @param categories - * Iterable over the category attributes - */ - protected void fillCategoriesMap(Iterable categories) - throws IOException { - categoriesMap.clear(); - - // for-each category - for (CategoryAttribute category : categories) { - // extracting the field-name to which this category belongs - String fieldName = indexingParams.getCategoryListParams( - category.getCategoryPath()).getTerm().field(); - - // getting the list of categories which belongs to that field - List list = categoriesMap.get(fieldName); - - // if no such list exists - if (list == null) { - // adding a new one to the map - list = new ArrayList(); - categoriesMap.put(fieldName, list); - } - - // adding the new category to the list - list.add(category.clone()); - } - } - - /** - * Get a category list tokenizer (or a series of such tokenizers) to create - * the category list tokens. - * - * @param categoryStream - * A stream containing {@link CategoryAttribute} with the - * relevant data. - * @return The category list tokenizer (or series of tokenizers) to be used - * in creating category list tokens. - */ - protected CategoryListTokenizer getCategoryListTokenizer( - TokenStream categoryStream) { - return getCountingListTokenizer(categoryStream); - } - - /** - * Get a {@link CountingListTokenizer} for creating counting list token. - * - * @param categoryStream - * A stream containing {@link CategoryAttribute}s with the - * relevant data. - * @return A counting list tokenizer to be used in creating counting list - * token. - */ - protected CountingListTokenizer getCountingListTokenizer( - TokenStream categoryStream) { - return new CountingListTokenizer(categoryStream, indexingParams); - } - - /** - * Get a {@link CategoryTokenizer} to create the category tokens. - * This method can be overridden for adding more attributes to the category - * tokens. - * - * @param categoryStream - * A stream containing {@link CategoryAttribute} with the - * relevant data. - * @return The {@link CategoryTokenizer} to be used in creating category - * tokens. - * @throws IOException If there is a low-level I/O error. - */ - protected CategoryTokenizer getCategoryTokenizer(TokenStream categoryStream) - throws IOException { - return new CategoryTokenizer(categoryStream, indexingParams); - } - - /** Adds the fields created in one of the "set" methods to the document */ - public Document build(Document doc) { - for (Field f : fieldList) { - doc.add(f); - } - return doc; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java (working copy) @@ -0,0 +1,181 @@ +package org.apache.lucene.facet.index; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map.Entry; + +import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy; +import org.apache.lucene.facet.index.params.CategoryListParams; +import org.apache.lucene.facet.index.params.FacetIndexingParams; +import org.apache.lucene.facet.taxonomy.CategoryPath; +import org.apache.lucene.facet.taxonomy.TaxonomyWriter; +import org.apache.lucene.facet.util.PartitionsUtils; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.UnsafeByteArrayOutputStream; +import org.apache.lucene.util.encoding.IntEncoder; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Builds a category list by encoding the category ordinals into one or more + * {@link BytesRef}. Each {@link BytesRef} corresponds to a set of ordinals that + * belong to the same partition. When partitions are not enabled (i.e. + * {@link FacetIndexingParams#getPartitionSize()} returns + * {@link Integer#MAX_VALUE}), only one {@link BytesRef} is returned by this + * class. + */ +public class CategoryListBuilder { + + /** Specializes encoding ordinals when partitions are enabled/disabled. */ + private static abstract class OrdinalsEncoder { + OrdinalsEncoder() {} + public abstract void encode(int ordinal); + public abstract HashMap finish(); + } + + private static final class NoPartitionsOrdinalsEncoder extends OrdinalsEncoder { + + private final IntEncoder encoder; + private final UnsafeByteArrayOutputStream ubaos; + private final String name; + + NoPartitionsOrdinalsEncoder(CategoryListParams categoryListParams) { + name = categoryListParams.getTerm().text(); + encoder = categoryListParams.createEncoder(); + ubaos = new UnsafeByteArrayOutputStream(); + encoder.reInit(ubaos); + } + + @Override + public void encode(int ordinal) { + try { + encoder.encode(ordinal); + } catch (IOException e) { + // shouldn't happen as we're writing to byte[] + throw new RuntimeException("unexpected exception", e); + } + } + + @Override + public HashMap finish() { + try { + encoder.close(); + } catch (IOException e) { + // shouldn't happen as we're writing to byte[] + throw new RuntimeException("unexpected exception", e); + } + HashMap result = new HashMap(); + result.put(name, new BytesRef(ubaos.toByteArray(), ubaos.getStartPos(), ubaos.length())); + return result; + } + + } + + private static final class PerPartitionOrdinalsEncoder extends OrdinalsEncoder { + + private final FacetIndexingParams indexingParams; + private final CategoryListParams categoryListParams; + private final int partitionSize; + private final HashMap partitionEncoder = new HashMap(); + private final HashMap partitionBytes = new HashMap(); + + PerPartitionOrdinalsEncoder(FacetIndexingParams indexingParams, CategoryListParams categoryListParams) { + this.indexingParams = indexingParams; + this.categoryListParams = categoryListParams; + this.partitionSize = indexingParams.getPartitionSize(); + } + + @Override + public void encode(int ordinal) { + final String name = PartitionsUtils.partitionNameByOrdinal(indexingParams, categoryListParams, ordinal); + IntEncoder encoder = partitionEncoder.get(name); + if (encoder == null) { + encoder = categoryListParams.createEncoder(); + final UnsafeByteArrayOutputStream ubaos = new UnsafeByteArrayOutputStream(); + encoder.reInit(ubaos); + partitionEncoder.put(name, encoder); + partitionBytes.put(name, ubaos); + } + try { + encoder.encode(ordinal % partitionSize); + } catch (IOException e) { + // shouldn't happen as we're writing to byte[] + throw new RuntimeException("unexpected exception", e); + } + } + + @Override + public HashMap finish() { + // finish encoding + IOUtils.closeWhileHandlingException(partitionEncoder.values()); + + HashMap bytes = new HashMap(); + for (Entry e : partitionBytes.entrySet()) { + UnsafeByteArrayOutputStream ubaos = e.getValue(); + bytes.put(e.getKey(), new BytesRef(ubaos.toByteArray(), ubaos.getStartPos(), ubaos.length())); + } + return bytes; + } + + } + + private final TaxonomyWriter taxoWriter; + private final OrdinalsEncoder ordinalsEncoder; + private final OrdinalPolicy ordinalPolicy; + + public CategoryListBuilder(CategoryListParams categoryListParams, FacetIndexingParams indexingParams, + TaxonomyWriter taxoWriter) { + this.taxoWriter = taxoWriter; + this.ordinalPolicy = indexingParams.getOrdinalPolicy(); + if (indexingParams.getPartitionSize() == Integer.MAX_VALUE) { + ordinalsEncoder = new NoPartitionsOrdinalsEncoder(categoryListParams); + } else { + ordinalsEncoder = new PerPartitionOrdinalsEncoder(indexingParams, categoryListParams); + } + } + + /** + * Encodes the given ordinal as well as any of its parent ordinals (per + * {@link OrdinalPolicy}). + */ + public void handle(int ordinal, CategoryPath cp) throws IOException { + ordinalsEncoder.encode(ordinal); + + // add all parent ordinals, per OrdinalPolicy + int parent = taxoWriter.getParent(ordinal); + while (parent > 0) { + if (ordinalPolicy.shouldAdd(parent)) { + ordinalsEncoder.encode(parent); + } + parent = taxoWriter.getParent(parent); + } + } + + /** + * Returns the encoded ordinals data. Every returned {@link BytesRef} + * corresponds to a single partition (as defined by + * {@link FacetIndexingParams#getPartitionSize()}) and the key denotes the + * partition ID. When no partitions are defined, the returned map includes + * only one value. + */ + public HashMap finish() { + return ordinalsEncoder.finish(); + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListBuilder.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListPayloadStream.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListPayloadStream.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/CategoryListPayloadStream.java (working copy) @@ -1,65 +0,0 @@ -package org.apache.lucene.facet.index; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; - -import org.apache.lucene.util.encoding.IntEncoder; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Accumulates category IDs for a single document, for writing in byte array - * form, for example, to a Lucene payload. - * - * @lucene.experimental - */ -public class CategoryListPayloadStream { - - private ByteArrayOutputStream baos = new ByteArrayOutputStream(50); - private IntEncoder encoder; - - /** Creates a payload stream using the specified encoder. */ - public CategoryListPayloadStream(IntEncoder encoder) { - this.encoder = encoder; - this.encoder.reInit(baos); - } - - /** Appends an integer to the stream. */ - public void appendIntToStream(int intValue) throws IOException { - encoder.encode(intValue); - } - - /** Returns the streamed bytes so far accumulated, as an array of bytes. */ - public byte[] convertStreamToByteArray() { - try { - encoder.close(); - return baos.toByteArray(); - } catch (IOException e) { - // This cannot happen, because of BAOS (no I/O). - return new byte[0]; - } - } - - /** Resets this stream to begin building a new payload. */ - public void reset() throws IOException { - encoder.close(); - baos.reset(); - encoder.reInit(baos); - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/DrillDownStream.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/DrillDownStream.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/index/DrillDownStream.java (working copy) @@ -0,0 +1,89 @@ +package org.apache.lucene.facet.index; + +import java.io.IOException; +import java.util.Iterator; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.facet.index.categorypolicy.PathPolicy; +import org.apache.lucene.facet.index.params.FacetIndexingParams; +import org.apache.lucene.facet.taxonomy.CategoryPath; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A {@link TokenStream} which creates category drill-down terms. + * + * @lucene.experimental + */ +public class DrillDownStream extends TokenStream { + + private final FacetIndexingParams indexingParams; + private final Iterator categories; + private final CharTermAttribute termAttribute; + private final PathPolicy pathPolicy; + + private CategoryPath current; + private boolean isParent; + + public DrillDownStream(Iterable categories, FacetIndexingParams indexingParams) { + termAttribute = addAttribute(CharTermAttribute.class); + this.categories = categories.iterator(); + this.indexingParams = indexingParams; + this.pathPolicy = indexingParams.getPathPolicy(); + } + + protected void addAdditionalAttributes(CategoryPath category, boolean isParent) { + // a hook for AssociationsDrillDownStream to add the associations payload to + // the drill-down terms + } + + @Override + public final boolean incrementToken() throws IOException { + if (current.length() == 0) { + if (!categories.hasNext()) { + return false; // no more categories + } + current = categories.next(); + termAttribute.resizeBuffer(current.charsNeededForFullPath()); + isParent = false; + } + + // copy current as drill-down term (it's either the leaf node or PathPolicy + // accepted it. + int nChars = indexingParams.drillDownTermText(current, termAttribute.buffer()); + termAttribute.setLength(nChars); + addAdditionalAttributes(current, isParent); + + // prepare current for next call by trimming the last component (parents) + do { + // skip all parent categories which are not accepted by PathPolicy + current.trim(1); + } while (!pathPolicy.shouldAdd(current) && current.length() > 0); + isParent = true; + return true; + } + + @Override + public void reset() throws IOException { + current = categories.next(); + termAttribute.resizeBuffer(current.charsNeededForFullPath()); + isParent = false; + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/index/DrillDownStream.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/index/FacetFields.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/FacetFields.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/facet/index/FacetFields.java (working copy) @@ -0,0 +1,214 @@ +package org.apache.lucene.facet.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.TextField; +import org.apache.lucene.facet.index.params.CategoryListParams; +import org.apache.lucene.facet.index.params.FacetIndexingParams; +import org.apache.lucene.facet.taxonomy.CategoryPath; +import org.apache.lucene.facet.taxonomy.TaxonomyWriter; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.util.BytesRef; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A utility class for adding facet fields to a document. Usually one field will + * be added for all facets, however per the + * {@link FacetIndexingParams#getCategoryListParams(CategoryPath)}, one field + * may be added for every group of facets. + * + * @lucene.experimental + */ +public class FacetFields { + + // a TokenStream for writing the counting list payload + private static final class CountingListStream extends TokenStream { + private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class); + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private Iterator> categoriesData; + + CountingListStream() {} + + @Override + public boolean incrementToken() throws IOException { + if (!categoriesData.hasNext()) { + return false; + } + + Entry entry = categoriesData.next(); + termAtt.setEmpty().append(entry.getKey()); + payloadAtt.setPayload(entry.getValue()); + return true; + } + + void setCategoriesData(HashMap categoriesData) { + this.categoriesData = categoriesData.entrySet().iterator(); + } + + } + + // The counting list is written in a payload, but we don't store it + // nor need norms. + private static final FieldType COUNTING_LIST_PAYLOAD_TYPE = new FieldType(); + static { + COUNTING_LIST_PAYLOAD_TYPE.setIndexed(true); + COUNTING_LIST_PAYLOAD_TYPE.setTokenized(true); + COUNTING_LIST_PAYLOAD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + COUNTING_LIST_PAYLOAD_TYPE.setStored(false); + COUNTING_LIST_PAYLOAD_TYPE.setOmitNorms(true); + COUNTING_LIST_PAYLOAD_TYPE.freeze(); + } + + // The drill-down field is added with a TokenStream, hence why it's based on + // TextField type. However in practice, it is added just like StringField. + // Therefore we set its IndexOptions to DOCS_ONLY. + private static final FieldType DRILL_DOWN_TYPE = new FieldType(TextField.TYPE_NOT_STORED); + static { + // TODO: once we cutover to DocValues, we can set it to DOCS_ONLY for this + // FacetFields (not associations) + DRILL_DOWN_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + DRILL_DOWN_TYPE.freeze(); + } + + protected final TaxonomyWriter taxonomyWriter; + + protected final FacetIndexingParams indexingParams; + + /** + * Constructs a new instance with the {@link FacetIndexingParams#ALL_PARENTS + * default} facet indexing params. + * + * @param taxonomyWriter + * used to resolve given categories to ordinals + */ + public FacetFields(TaxonomyWriter taxonomyWriter) { + this(taxonomyWriter, FacetIndexingParams.ALL_PARENTS); + } + + /** + * Constructs a new instance with the given facet indexing params. + * + * @param taxonomyWriter + * used to resolve given categories to ordinals + * @param params + * determines under which fields the categories should be indexed + */ + public FacetFields(TaxonomyWriter taxonomyWriter, FacetIndexingParams params) { + this.taxonomyWriter = taxonomyWriter; + this.indexingParams = params; + } + + /** + * Creates a mapping between a {@link CategoryListParams} and all + * {@link CategoryPath categories} that are associated with it. + */ + protected Map> createCategoryListMapping( + Iterable categories) { + HashMap> categoryLists = + new HashMap>(); + for (CategoryPath cp : categories) { + // each category may be indexed under a different field, so add it to the right list. + CategoryListParams clp = indexingParams.getCategoryListParams(cp); + List list = (List) categoryLists.get(clp); + if (list == null) { + list = new ArrayList(); + categoryLists.put(clp, list); + } + // DrillDownStream modifies the CategoryPath by calling trim(). That means + // that the source category, as the app ses it, is modified. While for + // most apps this is not a problem, we need to protect against it. If + // CategoryPath will be made immutable, we can stop cloning. + list.add(cp.clone()); + } + return categoryLists; + } + + /** Returns a {@link CategoryListBuilder} for encoding the given categories. */ + protected CategoryListBuilder getCategoryListBuilder(CategoryListParams categoryListParams, + Iterable categories /* needed for AssociationsFacetFields */) { + return new CategoryListBuilder(categoryListParams, indexingParams, taxonomyWriter); + } + + /** + * Returns a {@link DrillDownStream} for writing the categories drill-down + * terms. + */ + protected DrillDownStream getDrillDownStream(Iterable categories) { + return new DrillDownStream(categories, indexingParams); + } + + /** + * Returns the {@link FieldType} with which the drill-down terms should be + * indexed. The default is {@link IndexOptions#DOCS_ONLY}. + */ + protected FieldType fieldType() { + return DRILL_DOWN_TYPE; + } + + /** Adds the needed facet fields to the document. */ + public void addFields(Document doc, Iterable categories) throws IOException { + if (categories == null) { + throw new IllegalArgumentException("categories should not be null"); + } + + // TODO: add reuse capabilities to this class, per CLP objects: + // - drill-down field + // - counting list field + // - DrillDownStream + // - CountingListStream + + final Map> categoryLists = createCategoryListMapping(categories); + + // for each CLP we add a different field for drill-down terms as well as for + // counting list data. + for (Entry> e : categoryLists.entrySet()) { + final CategoryListParams clp = e.getKey(); + final String field = clp.getTerm().field(); + + // add the counting list data + CategoryListBuilder categoriesPayloadBuilder = getCategoryListBuilder(clp, e.getValue()); + for (CategoryPath cp : e.getValue()) { + int ordinal = taxonomyWriter.addCategory(cp); + categoriesPayloadBuilder.handle(ordinal , cp); + } + HashMap categoriesData = categoriesPayloadBuilder.finish(); + CountingListStream ts = new CountingListStream(); + ts.setCategoriesData(categoriesData); + doc.add(new Field(field, ts, COUNTING_LIST_PAYLOAD_TYPE)); + + // add the drill-down field + DrillDownStream drillDownStream = getDrillDownStream(e.getValue()); + Field drillDown = new Field(field, drillDownStream, fieldType()); + doc.add(drillDown); + } + } + +} Property changes on: lucene/facet/src/java/org/apache/lucene/facet/index/FacetFields.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttribute.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttribute.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttribute.java (working copy) @@ -1,129 +0,0 @@ -package org.apache.lucene.facet.index.attributes; - -import java.util.Collection; -import java.util.Set; - -import org.apache.lucene.util.Attribute; - -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An attribute which contains for a certain category the {@link CategoryPath} - * and additional properties. - * - * @lucene.experimental - */ -public interface CategoryAttribute extends Attribute { - - /** - * Set the content of this {@link CategoryAttribute} from another - * {@link CategoryAttribute} object. - * - * @param other - * The {@link CategoryAttribute} to take the content from. - */ - public void set(CategoryAttribute other); - - /** - * Sets the category path value of this attribute. - * - * @param cp - * A category path. May not be null. - */ - public void setCategoryPath(CategoryPath cp); - - /** - * Returns the value of this attribute: a category path. - * - * @return The category path last assigned to this attribute, or null if - * none has been assigned. - */ - public CategoryPath getCategoryPath(); - - /** - * Add a property. The property can be later retrieved using - * {@link #getProperty(Class)} with this property class .
- * Adding multiple properties of the same class is forbidden. - * - * @param property - * The property to add. - * @throws UnsupportedOperationException - * When attempting to add a property of a class that was added - * before and merge is prohibited. - */ - public void addProperty(CategoryProperty property) - throws UnsupportedOperationException; - - /** - * Get a property of a certain property class. - * - * @param propertyClass - * The required property class. - * @return The property of the given class, or null if no such property - * exists. - */ - public CategoryProperty getProperty( - Class propertyClass); - - /** - * Get a property of one of given property classes. - * - * @param propertyClasses - * The property classes. - * @return A property matching one of the given classes, or null if no such - * property exists. - */ - public CategoryProperty getProperty( - Collection> propertyClasses); - - /** - * Get all the active property classes. - * - * @return A set containing the active property classes, or {@code null} if - * there are no properties. - */ - public Set> getPropertyClasses(); - - /** - * Clone this {@link CategoryAttribute}. - * - * @return A clone of this {@link CategoryAttribute}. - */ - public CategoryAttribute clone(); - - /** - * Resets this attribute to its initial value: a null category path and no - * properties. - */ - public void clear(); - - /** - * Clear all properties. - */ - public void clearProperties(); - - /** - * Remove an property of a certain property class. - * - * @param propertyClass - * The required property class. - */ - public void remove(Class propertyClass); -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributeImpl.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributeImpl.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributeImpl.java (working copy) @@ -1,201 +0,0 @@ -package org.apache.lucene.facet.index.attributes; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Set; - -import org.apache.lucene.util.AttributeImpl; - -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An implementation of {@link CategoryAttribute}. - * - * @lucene.experimental - */ -public final class CategoryAttributeImpl extends AttributeImpl implements - CategoryAttribute { - - /** - * The category path instance. - */ - protected CategoryPath categoryPath; - - /** - * A map of properties associated to the current category path. - */ - protected HashMap, CategoryProperty> properties; - - /** - * Construct an empty CategoryAttributeImpl. - */ - public CategoryAttributeImpl() { - // do nothing - } - - /** - * Construct a CategoryAttributeImpl with the given CategoryPath. - * - * @param categoryPath - * The category path to use. - */ - public CategoryAttributeImpl(CategoryPath categoryPath) { - setCategoryPath(categoryPath); - } - - @Override - public void set(CategoryAttribute other) { - ((CategoryAttributeImpl) other).copyTo(this); - } - - /** - * Returns the category path value. - * - * @return The category path last assigned to this attribute, or null if - * none has been assigned. - */ - @Override - public CategoryPath getCategoryPath() { - return categoryPath; - } - - @Override - public void setCategoryPath(CategoryPath cp) { - categoryPath = cp; - } - - @Override - public void addProperty(CategoryProperty property) - throws UnsupportedOperationException { - if (properties == null) { - properties = new HashMap, CategoryProperty>(); - } - CategoryProperty existing = properties.get(property.getClass()); - if (existing == null) { - properties.put(property.getClass(), property); - } else { - existing.merge(property); - } - } - - @Override - public CategoryProperty getProperty( - Class propertyClass) { - if (properties == null) { - return null; - } - return properties.get(propertyClass); - } - - @Override - public CategoryProperty getProperty( - Collection> propertyClasses) { - if (properties == null) { - return null; - } - for (Class propertyClass : propertyClasses) { - CategoryProperty categoryProperty = properties.get(propertyClass); - if (categoryProperty != null) { - return categoryProperty; - } - } - return null; - } - - @Override - public void copyTo(AttributeImpl target) { - ((CategoryAttributeImpl) target).categoryPath = this.categoryPath; - ((CategoryAttributeImpl) target).properties = this.properties; - } - - @SuppressWarnings("unchecked") - @Override - public CategoryAttributeImpl clone() { - CategoryAttributeImpl ca = (CategoryAttributeImpl) super.clone(); - if (categoryPath != null) { - ca.categoryPath = categoryPath.clone(); - } - if (properties != null && !properties.isEmpty()) { - ca.properties = (HashMap, CategoryProperty>) properties - .clone(); - } - return ca; - } - - @Override - public void clear() { - categoryPath = null; - clearProperties(); - } - - @Override - public void clearProperties() { - if (properties != null) { - properties.clear(); - } - } - - @Override - public boolean equals(Object o) { - if (o == this) { - return true; - } - if (!(o instanceof CategoryAttributeImpl)) { - return false; - } - CategoryAttributeImpl other = (CategoryAttributeImpl) o; - if (categoryPath == null) { - return (other.categoryPath == null); - } - if (!categoryPath.equals(other.categoryPath)) { - return false; - } - if (properties == null || properties.isEmpty()) { - return (other.properties == null || other.properties.isEmpty()); - } - return properties.equals(other.properties); - } - - @Override - public int hashCode() { - if (categoryPath == null) { - return 0; - } - int hashCode = categoryPath.hashCode(); - if (properties != null && !properties.isEmpty()) { - hashCode ^= properties.hashCode(); - } - return hashCode; - } - - @Override - public Set> getPropertyClasses() { - if (properties == null || properties.isEmpty()) { - return null; - } - return properties.keySet(); - } - - @Override - public void remove(Class propertyClass) { - properties.remove(propertyClass); - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributesIterable.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributesIterable.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryAttributesIterable.java (working copy) @@ -1,73 +0,0 @@ -package org.apache.lucene.facet.index.attributes; - -import java.util.Iterator; - -import org.apache.lucene.facet.index.streaming.CategoryAttributesStream; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This class transforms an {@link Iterable} of {@link CategoryPath} objects - * into an {@link Iterable} of {@link CategoryAttribute} objects, which can be - * used to construct a {@link CategoryAttributesStream}. - * - * @lucene.experimental - */ -public class CategoryAttributesIterable implements Iterable { - - private Iterable inputIterable; - - public CategoryAttributesIterable(Iterable inputIterable) { - this.inputIterable = inputIterable; - } - - @Override - public Iterator iterator() { - return new CategoryAttributesIterator(this.inputIterable); - } - - private static class CategoryAttributesIterator implements Iterator { - - private Iterator internalIterator; - private CategoryAttributeImpl categoryAttributeImpl; - - public CategoryAttributesIterator(Iterable inputIterable) { - this.internalIterator = inputIterable.iterator(); - this.categoryAttributeImpl = new CategoryAttributeImpl(); - } - - @Override - public boolean hasNext() { - return this.internalIterator.hasNext(); - } - - @Override - public CategoryAttribute next() { - this.categoryAttributeImpl.setCategoryPath(this.internalIterator - .next()); - return this.categoryAttributeImpl; - } - - @Override - public void remove() { - this.internalIterator.remove(); - } - - } -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryProperty.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryProperty.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/attributes/CategoryProperty.java (working copy) @@ -1,51 +0,0 @@ -package org.apache.lucene.facet.index.attributes; - -import java.io.Serializable; - -import org.apache.lucene.facet.index.CategoryContainer; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Property that can be added to {@link CategoryAttribute}s during indexing. - * Note that properties are put in a map and could be shallow copied during - * {@link CategoryAttributeImpl#clone()}, therefore reuse of - * {@link CategoryProperty} objects is not recommended. Also extends - * {@link Serializable}, making the {@link CategoryContainer} serialization more - * elegant. - * - * @lucene.experimental - */ -public interface CategoryProperty extends Serializable { - - /** - * When adding categories with properties to a certain document, it is - * possible that the same category will be added more than once with - * different instances of the same property. This method defined how to - * treat such cases, by merging the newly added property into the one - * previously added. Implementing classes can assume that this method will - * be called only with a property of the same class. - * - * @param other - * The category property to merge. - * @throws UnsupportedOperationException - * If merging is prohibited for this property. - */ - public void merge(CategoryProperty other) - throws UnsupportedOperationException; -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/attributes/OrdinalProperty.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/attributes/OrdinalProperty.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/attributes/OrdinalProperty.java (working copy) @@ -1,72 +0,0 @@ -package org.apache.lucene.facet.index.attributes; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A {@link CategoryProperty} holding the ordinal from the taxonomy of the - * current category in {@link CategoryAttribute}. - *

- * Ordinal properties are added internally during processing of category - * streams, and it is recommended not to use it externally. - * - * @lucene.experimental - */ -public class OrdinalProperty implements CategoryProperty { - - protected int ordinal = -1; - - public int getOrdinal() { - return ordinal; - } - - public boolean hasBeenSet() { - return this.ordinal >= 0; - } - - public void setOrdinal(int value) { - this.ordinal = value; - } - - public void clear() { - this.ordinal = -1; - } - - @Override - public boolean equals(Object other) { - if (other == this) { - return true; - } - if (!(other instanceof OrdinalProperty)) { - return false; - } - OrdinalProperty o = (OrdinalProperty) other; - return o.ordinal == this.ordinal; - } - - @Override - public int hashCode() { - return this.ordinal; - } - - @Override - public void merge(CategoryProperty other) { - throw new UnsupportedOperationException( - "Merging ordinal attributes is prohibited"); - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/attributes/package.html =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/attributes/package.html (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/attributes/package.html (working copy) @@ -1,29 +0,0 @@ - - - -Category attributes and their properties for indexing - - -

Category attributes and their properties for indexing

- -Attributes for a {@link org.apache.lucene.facet.taxonomy.CategoryPath category}, -possibly containing -{@link org.apache.lucene.facet.index.attributes.CategoryProperty category property}'s. - - - \ No newline at end of file Index: lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/PathPolicy.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/PathPolicy.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/categorypolicy/PathPolicy.java (working copy) @@ -2,7 +2,7 @@ import java.io.Serializable; -import org.apache.lucene.facet.index.streaming.CategoryParentsStream; +import org.apache.lucene.facet.index.DrillDownStream; import org.apache.lucene.facet.taxonomy.CategoryPath; /* @@ -24,7 +24,7 @@ /** * Determines which {@link CategoryPath categories} should be added as terms to - * the {@link CategoryParentsStream}. The default approach is implemented by + * the {@link DrillDownStream}. The default approach is implemented by * {@link #ALL_CATEGORIES}. * * @lucene.experimental Index: lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetIndexingParams.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetIndexingParams.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetIndexingParams.java (working copy) @@ -130,7 +130,7 @@ public int getPartitionSize() { return partitionSize; } - + /** * Returns a list of all {@link CategoryListParams categoryListParams} that * are used for facets indexing. Index: lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetParamsMissingPropertyException.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetParamsMissingPropertyException.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/params/FacetParamsMissingPropertyException.java (working copy) @@ -1,32 +0,0 @@ -package org.apache.lucene.facet.index.params; - -import org.apache.lucene.facet.FacetException; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Thrown when the facets params are missing a property. * - * - * @lucene.experimental - */ -public class FacetParamsMissingPropertyException extends FacetException { - - public FacetParamsMissingPropertyException(String key) { - super("Property with key \"" + key + "\" not found"); - } -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryAttributesStream.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryAttributesStream.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryAttributesStream.java (working copy) @@ -1,80 +0,0 @@ -package org.apache.lucene.facet.index.streaming; - -import java.util.Iterator; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.index.attributes.CategoryAttribute; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An attribute stream built from an {@link Iterable} of - * {@link CategoryAttribute}. This stream should then be passed through several - * filters (see {@link CategoryParentsStream}, {@link CategoryListTokenizer} and - * {@link CategoryTokenizer}) until a token stream is produced that can be - * indexed by Lucene. - *

- * A CategoryAttributesStream object can be reused for producing more than one - * stream. To do that, the user should cause the underlying - * Iterable<CategoryAttribute> object to return a new set of categories, and - * then call {@link #reset()} to allow this stream to be used again. - * - * @lucene.experimental - */ -public class CategoryAttributesStream extends TokenStream { - - protected CategoryAttribute categoryAttribute; - - private Iterable iterable; - private Iterator iterator; - - /** - * Constructor - * - * @param iterable - * {@link Iterable} of {@link CategoryAttribute}, from which - * categories are taken. - */ - public CategoryAttributesStream(Iterable iterable) { - this.iterable = iterable; - this.iterator = null; - this.categoryAttribute = this.addAttribute(CategoryAttribute.class); - } - - @Override - public final boolean incrementToken() { - if (iterator == null) { - if (iterable == null) { - return false; - } - iterator = iterable.iterator(); - } - if (iterator.hasNext()) { - categoryAttribute.set(iterator.next()); - return true; - } - return false; - } - - @Override - public void reset() { - this.iterator = null; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryListTokenizer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryListTokenizer.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryListTokenizer.java (working copy) @@ -1,67 +0,0 @@ -package org.apache.lucene.facet.index.streaming; - -import java.io.IOException; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.index.params.FacetIndexingParams; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A base class for category list tokenizers, which add category list tokens to - * category streams. - * - * @lucene.experimental - */ -public abstract class CategoryListTokenizer extends CategoryTokenizerBase { - - /** - * @see CategoryTokenizerBase#CategoryTokenizerBase(TokenStream, FacetIndexingParams) - */ - public CategoryListTokenizer(TokenStream input, - FacetIndexingParams indexingParams) { - super(input, indexingParams); - } - - /** - * A method invoked once when the input stream begins, for subclass-specific - * processing. Subclass implementations must invoke this one, too! - */ - protected void handleStartOfInput() throws IOException { - // In this class, we do nothing. - } - - /** - * A method invoked once when the input stream ends, for subclass-specific - * processing. - */ - protected void handleEndOfInput() throws IOException { - // In this class, we do nothing. - } - - @Override - public void reset() throws IOException { - super.reset(); - handleStartOfInput(); - } - - @Override - public abstract boolean incrementToken() throws IOException; - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryParentsStream.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryParentsStream.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryParentsStream.java (working copy) @@ -1,186 +0,0 @@ -package org.apache.lucene.facet.index.streaming; - -import java.io.IOException; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -import org.apache.lucene.analysis.TokenFilter; - -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.index.attributes.OrdinalProperty; -import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy; -import org.apache.lucene.facet.index.categorypolicy.PathPolicy; -import org.apache.lucene.facet.index.params.FacetIndexingParams; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This class adds parents to a {@link CategoryAttributesStream}. The parents - * are added according to the {@link PathPolicy} and {@link OrdinalPolicy} from - * the {@link FacetIndexingParams} given in the constructor.
- * By default, category properties are removed when creating parents of a - * certain category. However, it is possible to retain certain property types - * using {@link #addRetainableProperty(Class)}. - * - * @lucene.experimental - */ -public class CategoryParentsStream extends TokenFilter { - - /** - * A {@link TaxonomyWriter} for adding categories and retrieving their - * ordinals. - */ - protected TaxonomyWriter taxonomyWriter; - - /** An attribute containing all data related to the category */ - protected CategoryAttribute categoryAttribute; - - /** A category property containing the category ordinal */ - protected OrdinalProperty ordinalProperty; - - /** - * A set of property classes that are to be retained when creating a parent - * token. - */ - private Set> retainableProperties; - - /** A {@link PathPolicy} for the category's parents' category paths. */ - private PathPolicy pathPolicy; - - /** An {@link OrdinalPolicy} for the category's parents' ordinals. */ - private OrdinalPolicy ordinalPolicy; - - /** - * Constructor. - * - * @param input - * The input stream to handle, must be derived from - * {@link CategoryAttributesStream}. - * @param taxonomyWriter - * The taxonomy writer to use for adding categories and - * retrieving their ordinals. - * @param indexingParams - * The indexing params used for filtering parents. - */ - public CategoryParentsStream(CategoryAttributesStream input, - TaxonomyWriter taxonomyWriter, FacetIndexingParams indexingParams) { - super(input); - this.categoryAttribute = this.addAttribute(CategoryAttribute.class); - this.taxonomyWriter = taxonomyWriter; - this.pathPolicy = indexingParams.getPathPolicy(); - this.ordinalPolicy = indexingParams.getOrdinalPolicy(); - this.ordinalPolicy.init(taxonomyWriter); - this.ordinalProperty = new OrdinalProperty(); - - } - - @Override - public final boolean incrementToken() throws IOException { - if (this.categoryAttribute.getCategoryPath() != null) { - // try adding the parent of the current category to the stream - clearCategoryProperties(); - boolean added = false; - // set the parent's ordinal, if illegal set -1 - int ordinal = this.ordinalProperty.getOrdinal(); - if (ordinal != -1) { - ordinal = this.taxonomyWriter.getParent(ordinal); - if (this.ordinalPolicy.shouldAdd(ordinal)) { - this.ordinalProperty.setOrdinal(ordinal); - try { - this.categoryAttribute.addProperty(ordinalProperty); - } catch (UnsupportedOperationException e) { - throw new IOException(e.getLocalizedMessage()); - } - added = true; - } else { - this.ordinalProperty.setOrdinal(-1); - } - } - // set the parent's category path, if illegal set null - CategoryPath cp = this.categoryAttribute.getCategoryPath(); - if (cp != null) { - cp.trim(1); - // if ordinal added, must also have category paths - if (added || this.pathPolicy.shouldAdd(cp)) { - this.categoryAttribute.setCategoryPath(cp); - added = true; - } else { - this.categoryAttribute.clear(); - } - } - if (added) { - // a legal parent exists - return true; - } - } - // no more parents - get new category - if (input.incrementToken()) { - int ordinal = taxonomyWriter.addCategory(this.categoryAttribute.getCategoryPath()); - this.ordinalProperty.setOrdinal(ordinal); - try { - this.categoryAttribute.addProperty(this.ordinalProperty); - } catch (UnsupportedOperationException e) { - throw new IOException(e.getLocalizedMessage()); - } - return true; - } - return false; - } - - /** - * Clear the properties of the current {@link CategoryAttribute} attribute - * before setting the parent attributes.
- * It is possible to retain properties of certain types the parent tokens, - * using {@link #addRetainableProperty(Class)}. - */ - protected void clearCategoryProperties() { - if (retainableProperties == null || retainableProperties.isEmpty()) { - categoryAttribute.clearProperties(); - } else { - List> propsToRemove = new LinkedList>(); - for (Class propertyClass : categoryAttribute.getPropertyClasses()) { - if (!retainableProperties.contains(propertyClass)) { - propsToRemove.add(propertyClass); - } - } - for (Class propertyClass : propsToRemove) { - categoryAttribute.remove(propertyClass); - } - } - } - - /** - * Add a {@link CategoryProperty} class which is retained when creating - * parent tokens. - * - * @param toRetain - * The property class to retain. - */ - public void addRetainableProperty(Class toRetain) { - if (this.retainableProperties == null) { - this.retainableProperties = new HashSet>(); - } - this.retainableProperties.add(toRetain); - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizer.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizer.java (working copy) @@ -1,67 +0,0 @@ -package org.apache.lucene.facet.index.streaming; - -import java.io.IOException; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; - -import org.apache.lucene.facet.index.params.FacetIndexingParams; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Basic class for setting the {@link CharTermAttribute}s and - * {@link PayloadAttribute}s of category tokens. - * - * @lucene.experimental - */ -public class CategoryTokenizer extends CategoryTokenizerBase { - - /** - * @see CategoryTokenizerBase#CategoryTokenizerBase(TokenStream, - * FacetIndexingParams) - */ - public CategoryTokenizer(TokenStream input, - FacetIndexingParams indexingParams) { - super(input, indexingParams); - } - - @Override - public final boolean incrementToken() throws IOException { - if (input.incrementToken()) { - if (categoryAttribute != null && categoryAttribute.getCategoryPath() != null) { - CategoryPath categoryPath = categoryAttribute.getCategoryPath(); - char[] termBuffer = termAttribute.resizeBuffer(categoryPath.charsNeededForFullPath()); - int nChars = indexingParams.drillDownTermText(categoryPath, termBuffer); - termAttribute.setLength(nChars); - setPayload(); - } - return true; - } - return false; - } - - /** - * Set the payload of the current category token. - */ - protected void setPayload() { - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizerBase.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizerBase.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CategoryTokenizerBase.java (working copy) @@ -1,78 +0,0 @@ -package org.apache.lucene.facet.index.streaming; - -import java.io.IOException; - -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.util.BytesRef; - -import org.apache.lucene.facet.index.CategoryDocumentBuilder; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.params.FacetIndexingParams; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * A base class for all token filters which add term and payload attributes to - * tokens and are to be used in {@link CategoryDocumentBuilder}. Contains three - * attributes: {@link CategoryAttribute}, {@link CharTermAttribute} and - * {@link PayloadAttribute}. - * - * @lucene.experimental - */ -public abstract class CategoryTokenizerBase extends TokenFilter { - - /** The stream's category attributes. */ - protected CategoryAttribute categoryAttribute; - - /** The stream's payload attribute. */ - protected PayloadAttribute payloadAttribute; - - /** The stream's term attribute. */ - protected CharTermAttribute termAttribute; - - /** The object used for constructing payloads. */ - protected BytesRef payload = new BytesRef(); - - /** Indexing params for creating term text **/ - protected FacetIndexingParams indexingParams; - - /** - * Constructor. - * - * @param input - * The input stream, either {@link CategoryParentsStream} or an - * extension of {@link CategoryTokenizerBase}. - * @param indexingParams - * The indexing params to use. - */ - public CategoryTokenizerBase(TokenStream input, - FacetIndexingParams indexingParams) { - super(input); - this.categoryAttribute = this.addAttribute(CategoryAttribute.class); - this.termAttribute = this.addAttribute(CharTermAttribute.class); - this.payloadAttribute = this.addAttribute(PayloadAttribute.class); - this.indexingParams = indexingParams; - } - - @Override - public abstract boolean incrementToken() throws IOException; - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CountingListTokenizer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CountingListTokenizer.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/streaming/CountingListTokenizer.java (working copy) @@ -1,127 +0,0 @@ -package org.apache.lucene.facet.index.streaming; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map.Entry; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.index.CategoryListPayloadStream; -import org.apache.lucene.facet.index.attributes.OrdinalProperty; -import org.apache.lucene.facet.index.params.CategoryListParams; -import org.apache.lucene.facet.index.params.FacetIndexingParams; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.util.PartitionsUtils; -import org.apache.lucene.util.encoding.IntEncoder; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * {@link CategoryListTokenizer} for facet counting - * - * @lucene.experimental - */ -public class CountingListTokenizer extends CategoryListTokenizer { - - /** A table for retrieving payload streams by category-list name. */ - protected HashMap payloadStreamsByName = - new HashMap(); - - /** An iterator over the payload streams */ - protected Iterator> payloadStreamIterator; - - public CountingListTokenizer(TokenStream input, - FacetIndexingParams indexingParams) { - super(input, indexingParams); - this.payloadStreamsByName = new HashMap(); - } - - @Override - protected void handleStartOfInput() throws IOException { - payloadStreamsByName.clear(); - payloadStreamIterator = null; - } - - @Override - public final boolean incrementToken() throws IOException { - if (input.incrementToken()) { - if (this.categoryAttribute != null) { - OrdinalProperty ordinalProperty = (OrdinalProperty) this.categoryAttribute - .getProperty(OrdinalProperty.class); - if (ordinalProperty != null && legalCategory()) { - CategoryPath categoryPath = this.categoryAttribute - .getCategoryPath(); - int ordinal = ordinalProperty.getOrdinal(); - CategoryListPayloadStream payloadStream = getPayloadStream( - categoryPath, ordinal); - int partitionSize = indexingParams.getPartitionSize(); - payloadStream.appendIntToStream(ordinal % partitionSize); - } - } - return true; - } - if (this.payloadStreamIterator == null) { - this.handleEndOfInput(); - this.payloadStreamIterator = this.payloadStreamsByName.entrySet() - .iterator(); - } - if (this.payloadStreamIterator.hasNext()) { - Entry entry = this.payloadStreamIterator - .next(); - String countingListName = entry.getKey(); - int length = countingListName.length(); - this.termAttribute.resizeBuffer(length); - countingListName.getChars(0, length, termAttribute.buffer(), 0); - this.termAttribute.setLength(length); - CategoryListPayloadStream payloadStream = entry.getValue(); - payload.bytes = payloadStream.convertStreamToByteArray(); - payload.offset = 0; - payload.length = payload.bytes.length; - this.payloadAttribute.setPayload(payload); - return true; - } - return false; - } - - /** - * A method which allows extending classes to filter the categories going - * into the counting list. - * - * @return By default returns {@code true}, meaning the current category is - * to be part of the counting list. For categories that should be - * filtered, return {@code false}. - */ - protected boolean legalCategory() { - return true; - } - - protected CategoryListPayloadStream getPayloadStream( - CategoryPath categoryPath, int ordinal) throws IOException { - CategoryListParams clParams = this.indexingParams.getCategoryListParams(categoryPath); - String name = PartitionsUtils.partitionNameByOrdinal(indexingParams, clParams, ordinal); - CategoryListPayloadStream fps = payloadStreamsByName.get(name); - if (fps == null) { - IntEncoder encoder = clParams.createEncoder(); - fps = new CategoryListPayloadStream(encoder); - payloadStreamsByName.put(name, fps); - } - return fps; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/index/streaming/package.html =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/index/streaming/package.html (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/index/streaming/package.html (working copy) @@ -1,35 +0,0 @@ - - - -Expert: attributes streaming definition for indexing facets - - -

Expert: attributes streaming definition for indexing facets

- -Steaming of facets attributes is a low level indexing interface with Lucene indexing. -There are two types of category related streams: -
    -
  • Category tokenizer stream handles tokenization for a single category, - e.g. for creating drill-down tokens.
    • -
  • Category list tokenizer stream handles tokenization for multiple categories, - e.g. for creating a counting list token, representing all the categories of - a certain document.
    • -
- - - \ No newline at end of file Index: lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationFloatSumAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationFloatSumAggregator.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationFloatSumAggregator.java (working copy) @@ -1,76 +0,0 @@ -package org.apache.lucene.facet.search.aggregator.association; - -import java.io.IOException; - -import org.apache.lucene.facet.enhancements.association.AssociationsPayloadIterator; -import org.apache.lucene.facet.index.params.CategoryListParams; -import org.apache.lucene.facet.search.aggregator.Aggregator; -import org.apache.lucene.index.IndexReader; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An {@link Aggregator} which updates the weight of a category by summing the - * weights of the float association it finds for every document. - * - * @lucene.experimental - */ -public class AssociationFloatSumAggregator implements Aggregator { - - protected final String field; - protected final float[] sumArray; - protected final AssociationsPayloadIterator associationsPayloadIterator; - - public AssociationFloatSumAggregator(IndexReader reader, float[] sumArray) throws IOException { - this(CategoryListParams.DEFAULT_TERM.field(), reader, sumArray); - } - - public AssociationFloatSumAggregator(String field, IndexReader reader, float[] sumArray) throws IOException { - this.field = field; - associationsPayloadIterator = new AssociationsPayloadIterator(reader, field); - this.sumArray = sumArray; - } - - @Override - public void aggregate(int ordinal) { - long association = associationsPayloadIterator.getAssociation(ordinal); - if (association != AssociationsPayloadIterator.NO_ASSOCIATION) { - sumArray[ordinal] += Float.intBitsToFloat((int) association); - } - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != this.getClass()) { - return false; - } - AssociationFloatSumAggregator that = (AssociationFloatSumAggregator) obj; - return that.field.equals(field) && that.sumArray == sumArray; - } - - @Override - public int hashCode() { - return field.hashCode(); - } - - @Override - public void setNextDoc(int docid, float score) throws IOException { - associationsPayloadIterator.setNextDoc(docid); - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationIntSumAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationIntSumAggregator.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/AssociationIntSumAggregator.java (working copy) @@ -1,76 +0,0 @@ -package org.apache.lucene.facet.search.aggregator.association; - -import java.io.IOException; - -import org.apache.lucene.facet.enhancements.association.AssociationsPayloadIterator; -import org.apache.lucene.facet.index.params.CategoryListParams; -import org.apache.lucene.facet.search.aggregator.Aggregator; -import org.apache.lucene.index.IndexReader; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An {@link Aggregator} which updates the weight of a category by summing the - * weights of the integer association it finds for every document. - * - * @lucene.experimental - */ -public class AssociationIntSumAggregator implements Aggregator { - - protected final String field; - protected final int[] sumArray; - protected final AssociationsPayloadIterator associationsPayloadIterator; - - public AssociationIntSumAggregator(IndexReader reader, int[] sumArray) throws IOException { - this(CategoryListParams.DEFAULT_TERM.field(), reader, sumArray); - } - - public AssociationIntSumAggregator(String field, IndexReader reader, int[] sumArray) throws IOException { - this.field = field; - associationsPayloadIterator = new AssociationsPayloadIterator(reader, field); - this.sumArray = sumArray; - } - - @Override - public void aggregate(int ordinal) { - long association = associationsPayloadIterator.getAssociation(ordinal); - if (association != AssociationsPayloadIterator.NO_ASSOCIATION) { - sumArray[ordinal] += association; - } - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != this.getClass()) { - return false; - } - AssociationIntSumAggregator that = (AssociationIntSumAggregator) obj; - return that.field.equals(field) && that.sumArray == sumArray; - } - - @Override - public int hashCode() { - return field.hashCode(); - } - - @Override - public void setNextDoc(int docid, float score) throws IOException { - associationsPayloadIterator.setNextDoc(docid); - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/package.html =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/package.html (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/association/package.html (working copy) @@ -1,22 +0,0 @@ - - - - -Association-based aggregators. - - Index: lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationFloatSumAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationFloatSumAggregator.java (working copy) +++ lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationFloatSumAggregator.java (working copy) @@ -1,8 +1,9 @@ -package org.apache.lucene.facet.search.aggregator.association; +package org.apache.lucene.facet.search.aggregator.associations; import java.io.IOException; -import org.apache.lucene.facet.enhancements.association.AssociationsPayloadIterator; +import org.apache.lucene.facet.associations.CategoryFloatAssociation; +import org.apache.lucene.facet.associations.FloatAssociationsPayloadIterator; import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.search.aggregator.Aggregator; import org.apache.lucene.index.IndexReader; @@ -25,8 +26,8 @@ */ /** - * An {@link Aggregator} which updates the weight of a category by summing the - * weights of the float association it finds for every document. + * An {@link Aggregator} which computes the weight of a category as the sum of + * the float values associated with it in the result documents. * * @lucene.experimental */ @@ -34,7 +35,7 @@ protected final String field; protected final float[] sumArray; - protected final AssociationsPayloadIterator associationsPayloadIterator; + protected final FloatAssociationsPayloadIterator associations; public AssociationFloatSumAggregator(IndexReader reader, float[] sumArray) throws IOException { this(CategoryListParams.DEFAULT_TERM.field(), reader, sumArray); @@ -42,15 +43,15 @@ public AssociationFloatSumAggregator(String field, IndexReader reader, float[] sumArray) throws IOException { this.field = field; - associationsPayloadIterator = new AssociationsPayloadIterator(reader, field); + associations = new FloatAssociationsPayloadIterator(reader, field, new CategoryFloatAssociation()); this.sumArray = sumArray; } @Override public void aggregate(int ordinal) { - long association = associationsPayloadIterator.getAssociation(ordinal); - if (association != AssociationsPayloadIterator.NO_ASSOCIATION) { - sumArray[ordinal] += Float.intBitsToFloat((int) association); + float association = associations.getAssociation(ordinal); + if (!Float.isNaN(association)) { + sumArray[ordinal] += association; } } @@ -70,7 +71,7 @@ @Override public void setNextDoc(int docid, float score) throws IOException { - associationsPayloadIterator.setNextDoc(docid); + associations.setNextDoc(docid); } } Index: lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationIntSumAggregator.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationIntSumAggregator.java (working copy) +++ lucene/facet/src/java/org/apache/lucene/facet/search/aggregator/associations/AssociationIntSumAggregator.java (working copy) @@ -1,8 +1,9 @@ -package org.apache.lucene.facet.search.aggregator.association; +package org.apache.lucene.facet.search.aggregator.associations; import java.io.IOException; -import org.apache.lucene.facet.enhancements.association.AssociationsPayloadIterator; +import org.apache.lucene.facet.associations.CategoryIntAssociation; +import org.apache.lucene.facet.associations.IntAssociationsPayloadIterator; import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.search.aggregator.Aggregator; import org.apache.lucene.index.IndexReader; @@ -25,8 +26,8 @@ */ /** - * An {@link Aggregator} which updates the weight of a category by summing the - * weights of the integer association it finds for every document. + * An {@link Aggregator} which computes the weight of a category as the sum of + * the integer values associated with it in the result documents. * * @lucene.experimental */ @@ -34,7 +35,7 @@ protected final String field; protected final int[] sumArray; - protected final AssociationsPayloadIterator associationsPayloadIterator; + protected final IntAssociationsPayloadIterator associations; public AssociationIntSumAggregator(IndexReader reader, int[] sumArray) throws IOException { this(CategoryListParams.DEFAULT_TERM.field(), reader, sumArray); @@ -42,14 +43,14 @@ public AssociationIntSumAggregator(String field, IndexReader reader, int[] sumArray) throws IOException { this.field = field; - associationsPayloadIterator = new AssociationsPayloadIterator(reader, field); + associations = new IntAssociationsPayloadIterator(reader, field, new CategoryIntAssociation()); this.sumArray = sumArray; } @Override public void aggregate(int ordinal) { - long association = associationsPayloadIterator.getAssociation(ordinal); - if (association != AssociationsPayloadIterator.NO_ASSOCIATION) { + long association = associations.getAssociation(ordinal); + if (association != IntAssociationsPayloadIterator.NO_ASSOCIATION) { sumArray[ordinal] += association; } } @@ -70,7 +71,7 @@ @Override public void setNextDoc(int docid, float score) throws IOException { - associationsPayloadIterator.setNextDoc(docid); + associations.setNextDoc(docid); } } Index: lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationFloatSumFacetRequest.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationFloatSumFacetRequest.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationFloatSumFacetRequest.java (working copy) @@ -1,70 +0,0 @@ -package org.apache.lucene.facet.search.params.association; - -import java.io.IOException; - -import org.apache.lucene.index.IndexReader; - -import org.apache.lucene.facet.search.FacetArrays; -import org.apache.lucene.facet.search.aggregator.Aggregator; -import org.apache.lucene.facet.search.aggregator.association.AssociationFloatSumAggregator; -import org.apache.lucene.facet.search.params.FacetRequest; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.TaxonomyReader; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Facet request for weighting facets according to their float association by - * summing the association values. - * - * @lucene.experimental - */ -public class AssociationFloatSumFacetRequest extends FacetRequest { - - /** - * Create a float association facet request for a given node in the - * taxonomy. - */ - public AssociationFloatSumFacetRequest(CategoryPath path, int num) { - super(path, num); - } - - @Override - public Aggregator createAggregator(boolean useComplements, - FacetArrays arrays, IndexReader reader, - TaxonomyReader taxonomy) throws IOException { - assert !useComplements : "complements are not supported by this FacetRequest"; - return new AssociationFloatSumAggregator(reader, arrays.getFloatArray()); - } - - @Override - public double getValueOf(FacetArrays arrays, int ordinal) { - return arrays.getFloatArray()[ordinal]; - } - - @Override - public boolean supportsComplements() { - return false; - } - - @Override - public boolean requireDocumentScore() { - return false; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationIntSumFacetRequest.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationIntSumFacetRequest.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/search/params/association/AssociationIntSumFacetRequest.java (working copy) @@ -1,70 +0,0 @@ -package org.apache.lucene.facet.search.params.association; - -import java.io.IOException; - -import org.apache.lucene.index.IndexReader; - -import org.apache.lucene.facet.search.FacetArrays; -import org.apache.lucene.facet.search.aggregator.Aggregator; -import org.apache.lucene.facet.search.aggregator.association.AssociationIntSumAggregator; -import org.apache.lucene.facet.search.params.FacetRequest; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.TaxonomyReader; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Facet request for weighting facets according to their integer association by - * summing the association values. - * - * @lucene.experimental - */ -public class AssociationIntSumFacetRequest extends FacetRequest { - - /** - * Create an integer association facet request for a given node in the - * taxonomy. - */ - public AssociationIntSumFacetRequest(CategoryPath path, int num) { - super(path, num); - } - - @Override - public Aggregator createAggregator(boolean useComplements, - FacetArrays arrays, IndexReader reader, - TaxonomyReader taxonomy) throws IOException { - assert !useComplements : "complements are not supported by this FacetRequest"; - return new AssociationIntSumAggregator(reader, arrays.getIntArray()); - } - - @Override - public double getValueOf(FacetArrays arrays, int ordinal) { - return arrays.getIntArray()[ordinal]; - } - - @Override - public boolean supportsComplements() { - return false; - } - - @Override - public boolean requireDocumentScore() { - return false; - } - -} Index: lucene/facet/src/java/org/apache/lucene/facet/search/params/association/package.html =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/params/association/package.html (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/facet/search/params/association/package.html (working copy) @@ -1,22 +0,0 @@ - - - - -Association-based Parameters for Faceted Search. - - Index: lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationFloatSumFacetRequest.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationFloatSumFacetRequest.java (working copy) +++ lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationFloatSumFacetRequest.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.facet.search.params.association; +package org.apache.lucene.facet.search.params.associations; import java.io.IOException; @@ -6,7 +6,7 @@ import org.apache.lucene.facet.search.FacetArrays; import org.apache.lucene.facet.search.aggregator.Aggregator; -import org.apache.lucene.facet.search.aggregator.association.AssociationFloatSumAggregator; +import org.apache.lucene.facet.search.aggregator.associations.AssociationFloatSumAggregator; import org.apache.lucene.facet.search.params.FacetRequest; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; @@ -29,8 +29,8 @@ */ /** - * Facet request for weighting facets according to their float association by - * summing the association values. + * A {@link FacetRequest} for weighting facets according to their float + * association by summing the association values. * * @lucene.experimental */ @@ -45,9 +45,8 @@ } @Override - public Aggregator createAggregator(boolean useComplements, - FacetArrays arrays, IndexReader reader, - TaxonomyReader taxonomy) throws IOException { + public Aggregator createAggregator(boolean useComplements, FacetArrays arrays, IndexReader reader, + TaxonomyReader taxonomy) throws IOException { assert !useComplements : "complements are not supported by this FacetRequest"; return new AssociationFloatSumAggregator(reader, arrays.getFloatArray()); } Index: lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationIntSumFacetRequest.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationIntSumFacetRequest.java (working copy) +++ lucene/facet/src/java/org/apache/lucene/facet/search/params/associations/AssociationIntSumFacetRequest.java (working copy) @@ -1,4 +1,4 @@ -package org.apache.lucene.facet.search.params.association; +package org.apache.lucene.facet.search.params.associations; import java.io.IOException; @@ -6,7 +6,7 @@ import org.apache.lucene.facet.search.FacetArrays; import org.apache.lucene.facet.search.aggregator.Aggregator; -import org.apache.lucene.facet.search.aggregator.association.AssociationIntSumAggregator; +import org.apache.lucene.facet.search.aggregator.associations.AssociationIntSumAggregator; import org.apache.lucene.facet.search.params.FacetRequest; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyReader; @@ -29,8 +29,8 @@ */ /** - * Facet request for weighting facets according to their integer association by - * summing the association values. + * A {@link FacetRequest} for weighting facets according to their integer + * association by summing the association values. * * @lucene.experimental */ @@ -45,9 +45,8 @@ } @Override - public Aggregator createAggregator(boolean useComplements, - FacetArrays arrays, IndexReader reader, - TaxonomyReader taxonomy) throws IOException { + public Aggregator createAggregator(boolean useComplements, FacetArrays arrays, IndexReader reader, + TaxonomyReader taxonomy) throws IOException { assert !useComplements : "complements are not supported by this FacetRequest"; return new AssociationIntSumAggregator(reader, arrays.getIntArray()); } Index: lucene/facet/src/java/org/apache/lucene/util/collections/IntToFloatMap.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/util/collections/IntToFloatMap.java (revision 0) +++ lucene/facet/src/java/org/apache/lucene/util/collections/IntToFloatMap.java (working copy) @@ -0,0 +1,631 @@ +package org.apache.lucene.util.collections; + +import java.util.Arrays; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * An Array-based hashtable which maps primitive int to a primitive float.
+ * The hashtable is constracted with a given capacity, or 16 as a default. In + * case there's not enough room for new pairs, the hashtable grows.
+ * Capacity is adjusted to a power of 2, and there are 2 * capacity entries for + * the hash. + * + * The pre allocated arrays (for keys, values) are at length of capacity + 1, + * when index 0 is used as 'Ground' or 'NULL'.
+ * + * The arrays are allocated ahead of hash operations, and form an 'empty space' + * list, to which the key,value pair is allocated. + * + * @lucene.experimental + */ +public class IntToFloatMap { + + public static final float GROUND = Float.NaN; + + /** + * Implements an IntIterator which iterates over all the allocated indexes. + */ + private final class IndexIterator implements IntIterator { + /** + * The last used baseHashIndex. Needed for "jumping" from one hash entry + * to another. + */ + private int baseHashIndex = 0; + + /** + * The next not-yet-visited index. + */ + private int index = 0; + + /** + * Index of the last visited pair. Used in {@link #remove()}. + */ + private int lastIndex = 0; + + /** + * Create the Iterator, make index point to the "first" + * index which is not empty. If such does not exist (eg. the map is + * empty) it would be zero. + */ + public IndexIterator() { + for (baseHashIndex = 0; baseHashIndex < baseHash.length; ++baseHashIndex) { + index = baseHash[baseHashIndex]; + if (index != 0) { + break; + } + } + } + + @Override + public boolean hasNext() { + return (index != 0); + } + + @Override + public int next() { + // Save the last index visited + lastIndex = index; + + // next the index + index = next[index]; + + // if the next index points to the 'Ground' it means we're done with + // the current hash entry and we need to jump to the next one. This + // is done until all the hash entries had been visited. + while (index == 0 && ++baseHashIndex < baseHash.length) { + index = baseHash[baseHashIndex]; + } + + return lastIndex; + } + + @Override + public void remove() { + IntToFloatMap.this.remove(keys[lastIndex]); + } + + } + + /** + * Implements an IntIterator, used for iteration over the map's keys. + */ + private final class KeyIterator implements IntIterator { + private IntIterator iterator = new IndexIterator(); + + KeyIterator() { } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public int next() { + return keys[iterator.next()]; + } + + @Override + public void remove() { + iterator.remove(); + } + } + + /** + * Implements an Iterator of a generic type T used for iteration over the + * map's values. + */ + private final class ValueIterator implements FloatIterator { + private IntIterator iterator = new IndexIterator(); + + ValueIterator() { } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public float next() { + return values[iterator.next()]; + } + + @Override + public void remove() { + iterator.remove(); + } + } + + /** + * Default capacity - in case no capacity was specified in the constructor + */ + private static int defaultCapacity = 16; + + /** + * Holds the base hash entries. if the capacity is 2^N, than the base hash + * holds 2^(N+1). It can hold + */ + int[] baseHash; + + /** + * The current capacity of the map. Always 2^N and never less than 16. We + * never use the zero index. It is needed to improve performance and is also + * used as "ground". + */ + private int capacity; + /** + * All objects are being allocated at map creation. Those objects are "free" + * or empty. Whenever a new pair comes along, a pair is being "allocated" or + * taken from the free-linked list. as this is just a free list. + */ + private int firstEmpty; + + /** + * hashFactor is always (2^(N+1)) - 1. Used for faster hashing. + */ + private int hashFactor; + + /** + * This array holds the unique keys + */ + int[] keys; + + /** + * In case of collisions, we implement a float linked list of the colliding + * hash's with the following next[] and prev[]. Those are also used to store + * the "empty" list. + */ + int[] next; + + private int prev; + + /** + * Number of currently objects in the map. + */ + private int size; + + /** + * This array holds the values + */ + float[] values; + + /** + * Constructs a map with default capacity. + */ + public IntToFloatMap() { + this(defaultCapacity); + } + + /** + * Constructs a map with given capacity. Capacity is adjusted to a native + * power of 2, with minimum of 16. + * + * @param capacity + * minimum capacity for the map. + */ + public IntToFloatMap(int capacity) { + this.capacity = 16; + // Minimum capacity is 16.. + while (this.capacity < capacity) { + // Multiply by 2 as long as we're still under the requested capacity + this.capacity <<= 1; + } + + // As mentioned, we use the first index (0) as 'Ground', so we need the + // length of the arrays to be one more than the capacity + int arrayLength = this.capacity + 1; + + this.values = new float[arrayLength]; + this.keys = new int[arrayLength]; + this.next = new int[arrayLength]; + + // Hash entries are twice as big as the capacity. + int baseHashSize = this.capacity << 1; + + this.baseHash = new int[baseHashSize]; + + this.values[0] = GROUND; + + // The has factor is 2^M - 1 which is used as an "AND" hashing operator. + // {@link #calcBaseHash()} + this.hashFactor = baseHashSize - 1; + + this.size = 0; + + clear(); + } + + /** + * Adds a pair to the map. Takes the first empty position from the + * empty-linked-list's head - {@link #firstEmpty}. + * + * New pairs are always inserted to baseHash, and are followed by the old + * colliding pair. + * + * @param key + * integer which maps the given Object + * @param v + * float value which is being mapped using the given key + */ + private void prvt_put(int key, float v) { + // Hash entry to which the new pair would be inserted + int hashIndex = calcBaseHashIndex(key); + + // 'Allocating' a pair from the "Empty" list. + int objectIndex = firstEmpty; + + // Setting data + firstEmpty = next[firstEmpty]; + values[objectIndex] = v; + keys[objectIndex] = key; + + // Inserting the new pair as the first node in the specific hash entry + next[objectIndex] = baseHash[hashIndex]; + baseHash[hashIndex] = objectIndex; + + // Announcing a new pair was added! + ++size; + } + + /** + * Calculating the baseHash index using the internal hashFactor + * . + */ + protected int calcBaseHashIndex(int key) { + return key & hashFactor; + } + + /** + * Empties the map. Generates the "Empty" space list for later allocation. + */ + public void clear() { + // Clears the hash entries + Arrays.fill(this.baseHash, 0); + + // Set size to zero + size = 0; + + // Mark all array entries as empty. This is done with + // firstEmpty pointing to the first valid index (1 as 0 is + // used as 'Ground'). + firstEmpty = 1; + + // And setting all the next[i] to point at + // i+1. + for (int i = 1; i < this.capacity;) { + next[i] = ++i; + } + + // Surly, the last one should point to the 'Ground'. + next[this.capacity] = 0; + } + + /** + * Checks if a given key exists in the map. + * + * @param key + * that is checked against the map data. + * @return true if the key exists in the map. false otherwise. + */ + public boolean containsKey(int key) { + return find(key) != 0; + } + + /** + * Checks if the given value exists in the map.
+ * This method iterates over the collection, trying to find an equal object. + * + * @param value + * float value that is checked against the map data. + * @return true if the value exists in the map, false otherwise. + */ + public boolean containsValue(float value) { + for (FloatIterator iterator = iterator(); iterator.hasNext();) { + float d = iterator.next(); + if (d == value) { + return true; + } + } + return false; + } + + /** + * Find the actual index of a given key. + * + * @return index of the key. zero if the key wasn't found. + */ + protected int find(int key) { + // Calculate the hash entry. + int baseHashIndex = calcBaseHashIndex(key); + + // Start from the hash entry. + int localIndex = baseHash[baseHashIndex]; + + // while the index does not point to the 'Ground' + while (localIndex != 0) { + // returns the index found in case of of a matching key. + if (keys[localIndex] == key) { + return localIndex; + } + + // next the local index + localIndex = next[localIndex]; + } + + // If we got this far, it could only mean we did not find the key we + // were asked for. return 'Ground' index. + return 0; + } + + /** + * Find the actual index of a given key with it's baseHashIndex.
+ * Some methods use the baseHashIndex. If those call {@link #find} there's + * no need to re-calculate that hash. + * + * @return the index of the given key, or 0 as 'Ground' if the key wasn't + * found. + */ + private int findForRemove(int key, int baseHashIndex) { + // Start from the hash entry. + this.prev = 0; + int index = baseHash[baseHashIndex]; + + // while the index does not point to the 'Ground' + while (index != 0) { + // returns the index found in case of of a matching key. + if (keys[index] == key) { + return index; + } + + // next the local index + prev = index; + index = next[index]; + } + + // If we got this far, it could only mean we did not find the key we + // were asked for. return 'Ground' index. + this.prev = 0; + return 0; + } + + /** + * Returns the value mapped with the given key. + * + * @param key + * int who's mapped object we're interested in. + * @return a float value mapped by the given key. float.NaN if the key wasn't found. + */ + public float get(int key) { + return values[find(key)]; + } + + /** + * Grows the map. Allocates a new map of float the capacity, and + * fast-insert the old key-value pairs. + */ + protected void grow() { + IntToFloatMap that = new IntToFloatMap( + this.capacity * 2); + + // Iterates fast over the collection. Any valid pair is put into the new + // map without checking for duplicates or if there's enough space for + // it. + for (IndexIterator iterator = new IndexIterator(); iterator.hasNext();) { + int index = iterator.next(); + that.prvt_put(this.keys[index], this.values[index]); + } + + // Copy that's data into this. + this.capacity = that.capacity; + this.size = that.size; + this.firstEmpty = that.firstEmpty; + this.values = that.values; + this.keys = that.keys; + this.next = that.next; + this.baseHash = that.baseHash; + this.hashFactor = that.hashFactor; + } + + /** + * + * @return true if the map is empty. false otherwise. + */ + public boolean isEmpty() { + return size == 0; + } + + /** + * Returns a new iterator for the mapped float values. + */ + public FloatIterator iterator() { + return new ValueIterator(); + } + + /** Returns an iterator on the map keys. */ + public IntIterator keyIterator() { + return new KeyIterator(); + } + + /** + * Prints the baseHash array, used for debug purposes. + */ + @SuppressWarnings("unused") + private String getBaseHashAsString() { + return Arrays.toString(this.baseHash); + } + + /** + * Inserts the <key,value> pair into the map. If the key already exists, + * this method updates the mapped value to the given one, returning the old + * mapped value. + * + * @return the old mapped value, or {@link Float#NaN} if the key didn't exist. + */ + public float put(int key, float v) { + // Does key exists? + int index = find(key); + + // Yes! + if (index != 0) { + // Set new data and exit. + float old = values[index]; + values[index] = v; + return old; + } + + // Is there enough room for a new pair? + if (size == capacity) { + // No? Than grow up! + grow(); + } + + // Now that everything is set, the pair can be just put inside with no + // worries. + prvt_put(key, v); + + return Float.NaN; + } + + /** + * Removes a <key,value> pair from the map and returns the mapped value, + * or {@link Float#NaN} if the none existed. + * + * @param key used to find the value to remove + * @return the removed value or {@link Float#NaN} if none existed. + */ + public float remove(int key) { + int baseHashIndex = calcBaseHashIndex(key); + int index = findForRemove(key, baseHashIndex); + if (index != 0) { + // If it is the first in the collision list, we should promote its + // next colliding element. + if (prev == 0) { + baseHash[baseHashIndex] = next[index]; + } + + next[prev] = next[index]; + next[index] = firstEmpty; + firstEmpty = index; + --size; + return values[index]; + } + + return Float.NaN; + } + + /** + * @return number of pairs currently in the map + */ + public int size() { + return this.size; + } + + /** + * Translates the mapped pairs' values into an array of Objects + * + * @return a float array of all the values currently in the map. + */ + public float[] toArray() { + int j = -1; + float[] array = new float[size]; + + // Iterates over the values, adding them to the array. + for (FloatIterator iterator = iterator(); iterator.hasNext();) { + array[++j] = iterator.next(); + } + return array; + } + + /** + * Translates the mapped pairs' values into an array of T + * + * @param a + * the array into which the elements of the list are to be + * stored. If it is big enough use whatever space we need, + * setting the one after the true data as {@link Float#NaN}. + * + * @return an array containing the elements of the list, using the given + * parameter if big enough, otherwise allocate an appropriate array + * and return it. + * + */ + public float[] toArray(float[] a) { + int j = 0; + if (a.length < this.size()) { + a = new float[this.size()]; + } + + // Iterates over the values, adding them to the array. + for (FloatIterator iterator = iterator(); iterator.hasNext(); ++j) { + a[j] = iterator.next(); + } + + if (j < a.length) { + a[j] = Float.NaN; + } + + return a; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append('{'); + IntIterator keyIterator = keyIterator(); + while (keyIterator.hasNext()) { + int key = keyIterator.next(); + sb.append(key); + sb.append('='); + sb.append(get(key)); + if (keyIterator.hasNext()) { + sb.append(','); + sb.append(' '); + } + } + sb.append('}'); + return sb.toString(); + } + + @Override + public int hashCode() { + return getClass().hashCode() ^ size(); + } + + @Override + public boolean equals(Object o) { + IntToFloatMap that = (IntToFloatMap)o; + if (that.size() != this.size()) { + return false; + } + + IntIterator it = keyIterator(); + while (it.hasNext()) { + int key = it.next(); + if (!that.containsKey(key)) { + return false; + } + + float v1 = this.get(key); + float v2 = that.get(key); + if (Float.compare(v1, v2) != 0) { + return false; + } + } + return true; + } +} \ No newline at end of file Property changes on: lucene/facet/src/java/org/apache/lucene/util/collections/IntToFloatMap.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java (revision 1427505) +++ lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java (working copy) @@ -1,5 +1,6 @@ package org.apache.lucene.util.encoding; +import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; @@ -43,7 +44,7 @@ * * @lucene.experimental */ -public abstract class IntEncoder { +public abstract class IntEncoder implements Closeable { protected OutputStream out = null; @@ -68,6 +69,7 @@ * NOTE: overriding classes should make sure they either call * super.close() or close the output stream themselves. */ + @Override public void close() throws IOException { if (out != null) { out.close(); Index: lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (working copy) @@ -15,7 +15,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.search.params.FacetRequest; @@ -255,9 +255,8 @@ protected final void indexDoc(FacetIndexingParams iParams, RandomIndexWriter iw, TaxonomyWriter tw, String content, List categories) throws IOException { Document d = new Document(); - CategoryDocumentBuilder builder = new CategoryDocumentBuilder(tw, iParams); - builder.setCategoryPaths(categories); - builder.build(d); + FacetFields facetFields = new FacetFields(tw, iParams); + facetFields.addFields(d, categories); d.add(new TextField("content", content, Field.Store.YES)); iw.addDocument(d); } Index: lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java (working copy) @@ -2,13 +2,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.search.FacetsCollector; import org.apache.lucene.facet.search.params.CountFacetRequest; @@ -117,11 +118,9 @@ public static void add(FacetIndexingParams iParams, RandomIndexWriter iw, TaxonomyWriter tw, String... strings) throws IOException { - ArrayList cps = new ArrayList(); - CategoryPath cp = new CategoryPath(strings); - cps.add(cp); Document d = new Document(); - new CategoryDocumentBuilder(tw, iParams).setCategoryPaths(cps).build(d); + FacetFields facetFields = new FacetFields(tw, iParams); + facetFields.addFields(d, Collections.singletonList(new CategoryPath(strings))); d.add(new TextField("content", "alpha", Field.Store.YES)); iw.addDocument(d); } Index: lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy1.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy1.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy1.java (working copy) @@ -1,76 +0,0 @@ -package org.apache.lucene.facet.enhancements; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.enhancements.CategoryEnhancement; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryEnhancementDummy1 implements CategoryEnhancement { - - @Override - public boolean generatesCategoryList() { - return false; - } - - @Override - public String getCategoryListTermText() { - return null; - } - - @Override - public CategoryListTokenizer getCategoryListTokenizer( - TokenStream tokenizer, EnhancementsIndexingParams indexingParams, - TaxonomyWriter taxonomyWriter) { - return null; - } - - @Override - public byte[] getCategoryTokenBytes(CategoryAttribute categoryAttribute) { - return null; - } - - @Override - public Object extractCategoryTokenData(byte[] buffer, int offset, int length) { - return null; - } - - @Override - public CategoryProperty getRetainableProperty() { - return null; - } - - @Override - public boolean equals(Object o) { - if (o instanceof CategoryEnhancementDummy1) { - return true; - } - return false; - } - - @Override - public int hashCode() { - return super.hashCode(); - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy2.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy2.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy2.java (working copy) @@ -1,85 +0,0 @@ -package org.apache.lucene.facet.enhancements; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.enhancements.CategoryEnhancement; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.DummyProperty; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryEnhancementDummy2 implements CategoryEnhancement { - - public static byte[] CATEGORY_TOKEN_BYTES = new byte[] { 3, 0, 7 }; - - @Override - public boolean generatesCategoryList() { - return false; - } - - @Override - public String getCategoryListTermText() { - return null; - } - - @Override - public CategoryListTokenizer getCategoryListTokenizer( - TokenStream tokenizer, EnhancementsIndexingParams indexingParams, - TaxonomyWriter taxonomyWriter) { - return null; - } - - @Override - public byte[] getCategoryTokenBytes(CategoryAttribute categoryAttribute) { - return CATEGORY_TOKEN_BYTES; - } - - @Override - public Object extractCategoryTokenData(byte[] buffer, int offset, int length) { - if (length != CATEGORY_TOKEN_BYTES.length) { - throw new IllegalArgumentException("unexpected data length " - + length); - } - byte[] ret = new byte[length]; - System.arraycopy(buffer, offset, ret, 0, length); - return ret; - } - - @Override - public CategoryProperty getRetainableProperty() { - return DummyProperty.INSTANCE; - } - - @Override - public boolean equals(Object o) { - if (o instanceof CategoryEnhancementDummy2) { - return true; - } - return false; - } - - @Override - public int hashCode() { - return super.hashCode(); - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy3.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy3.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/enhancements/CategoryEnhancementDummy3.java (working copy) @@ -1,84 +0,0 @@ -package org.apache.lucene.facet.enhancements; - -import org.apache.lucene.analysis.TokenStream; - -import org.apache.lucene.facet.enhancements.CategoryEnhancement; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.index.streaming.CategoryListTokenizer; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryEnhancementDummy3 implements CategoryEnhancement { - - public static byte[] CATEGORY_TOKEN_BYTES = new byte[] { 5, -1, 33, 8 }; - - @Override - public boolean generatesCategoryList() { - return false; - } - - @Override - public String getCategoryListTermText() { - return null; - } - - @Override - public CategoryListTokenizer getCategoryListTokenizer( - TokenStream tokenizer, EnhancementsIndexingParams indexingParams, - TaxonomyWriter taxonomyWriter) { - return null; - } - - @Override - public byte[] getCategoryTokenBytes(CategoryAttribute categoryAttribute) { - return CATEGORY_TOKEN_BYTES; - } - - @Override - public Object extractCategoryTokenData(byte[] buffer, int offset, int length) { - if (length != CATEGORY_TOKEN_BYTES.length) { - throw new IllegalArgumentException("unexpected data length " - + length); - } - byte[] ret = new byte[length]; - System.arraycopy(buffer, offset, ret, 0, length); - return ret; - } - - @Override - public CategoryProperty getRetainableProperty() { - return null; - } - - @Override - public boolean equals(Object o) { - if (o instanceof CategoryEnhancementDummy3) { - return true; - } - return false; - } - - @Override - public int hashCode() { - return super.hashCode(); - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/enhancements/EnhancementsPayloadIteratorTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/enhancements/EnhancementsPayloadIteratorTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/enhancements/EnhancementsPayloadIteratorTest.java (working copy) @@ -1,113 +0,0 @@ -package org.apache.lucene.facet.enhancements; - -import java.io.IOException; - -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.Directory; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.facet.enhancements.EnhancementsPayloadIterator; -import org.apache.lucene.facet.enhancements.association.AssociationEnhancement; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.example.association.AssociationIndexer; -import org.apache.lucene.facet.example.association.AssociationUtils; -import org.apache.lucene.facet.search.DrillDown; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class EnhancementsPayloadIteratorTest extends LuceneTestCase { - - private static Directory indexDir; - private static Directory taxoDir; - private static EnhancementsIndexingParams indexingParams; - private static AssociationEnhancement associationEnhancement; - - @BeforeClass - public static void buildAssociationIndex() throws Exception { - // create Directories for the search index and for the taxonomy index - indexDir = newDirectory(); - taxoDir = newDirectory(); - - // index the sample documents - if (VERBOSE) { - System.out.println("index the sample documents..."); - } - AssociationIndexer.index(indexDir, taxoDir); - - indexingParams = AssociationUtils.assocIndexingParams; - associationEnhancement = (AssociationEnhancement) indexingParams - .getCategoryEnhancements().get(0); - } - - @Test - public void testFullIterator() throws IOException { - IndexReader indexReader = DirectoryReader.open(indexDir); - Term term = DrillDown.term(indexingParams, new CategoryPath("tags", "lucene")); - EnhancementsPayloadIterator iterator = new EnhancementsPayloadIterator( - indexingParams.getCategoryEnhancements(), indexReader, term); - assertTrue("Unexpected failure of init()", iterator.init()); - assertTrue("Missing instance of tags/lucene in doc 0", iterator.setdoc(0)); - int assoc = (Integer) iterator.getCategoryData(associationEnhancement); - assertEquals("Unexpected association value for tags/lucene in doc 0", 3, assoc, 1E-5); - assertTrue("Missing instance of tags/lucene in doc 1", iterator.setdoc(1)); - assoc = (Integer) iterator.getCategoryData(associationEnhancement); - assertEquals("Unexpected association value for tags/lucene in doc 1", 1, assoc, 1E-5); - indexReader.close(); - } - - @Test - public void testEmptyIterator() throws IOException { - IndexReader indexReader = DirectoryReader.open(indexDir); - Term term = DrillDown.term(indexingParams, new CategoryPath("root","a", "f2")); - EnhancementsPayloadIterator iterator = new EnhancementsPayloadIterator( - indexingParams.getCategoryEnhancements(), indexReader, term); - assertTrue("Unexpected failure of init()", iterator.init()); - assertFalse("Unexpected payload for root/a/f2 in doc 0", iterator.setdoc(0)); - assertFalse("Unexpected instance of root/a/f2 in doc 1", iterator.setdoc(1)); - indexReader.close(); - } - - @Test - public void testPartialIterator() throws IOException { - IndexReader indexReader = DirectoryReader.open(indexDir); - Term term = DrillDown.term(indexingParams, new CategoryPath("genre","software")); - EnhancementsPayloadIterator iterator = new EnhancementsPayloadIterator( - indexingParams.getCategoryEnhancements(), indexReader, term); - assertTrue("Unexpected failure of init()", iterator.init()); - assertFalse("Unexpected payload for genre/computing in doc 0", iterator.setdoc(0)); - assertTrue("Missing instance of genre/computing in doc 1", iterator.setdoc(1)); - float assoc = Float.intBitsToFloat((Integer) iterator - .getCategoryData(associationEnhancement)); - assertEquals("Unexpected association value for genre/computing in doc 1", 0.34f, assoc, 0.001); - indexReader.close(); - } - - @AfterClass - public static void closeDirectories() throws IOException { - indexDir.close(); - indexDir = null; - taxoDir.close(); - taxoDir = null; - } -} Index: lucene/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/enhancements/TwoEnhancementsTest.java (working copy) @@ -1,128 +0,0 @@ -package org.apache.lucene.facet.enhancements; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.document.Document; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.search.DrillDown; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; -import org.junit.Test; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class TwoEnhancementsTest extends LuceneTestCase { - - @Test - public void testTwoEmptyAndNonEmptyByteArrays() throws Exception { - Directory indexDir = newDirectory(); - Directory taxoDir = newDirectory(); - - EnhancementsIndexingParams indexingParams = new EnhancementsIndexingParams( - new CategoryEnhancementDummy1(), new CategoryEnhancementDummy3()); - - // add document with a category containing data for both enhancements - List categoryPaths = new ArrayList(); - categoryPaths.add(new CategoryPath("a", "b")); - - RandomIndexWriter indexWriter = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))); - TaxonomyWriter taxo = new DirectoryTaxonomyWriter(taxoDir); - - // a category document builder will add the categories to a document - // once build() is called - Document doc = new Document(); - indexWriter.addDocument(new EnhancementsDocumentBuilder(taxo, - indexingParams).setCategoryPaths(categoryPaths).build(doc)); - - IndexReader indexReader = indexWriter.getReader(); - indexWriter.close(); - - Term term = DrillDown.term(indexingParams, new CategoryPath("a","b")); - EnhancementsPayloadIterator iterator = new EnhancementsPayloadIterator( - indexingParams.getCategoryEnhancements(), indexReader, term); - - assertTrue("EnhancementsPayloadIterator failure", iterator.init()); - assertTrue("Missing document 0", iterator.setdoc(0)); - assertNull("Unexpected data for CategoryEnhancementDummy2", iterator - .getCategoryData(new CategoryEnhancementDummy1())); - byte[] dummy3 = (byte[]) iterator - .getCategoryData(new CategoryEnhancementDummy3()); - assertTrue("Bad array returned for CategoryEnhancementDummy3", Arrays - .equals(dummy3, CategoryEnhancementDummy3.CATEGORY_TOKEN_BYTES)); - indexReader.close(); - indexDir.close(); - taxo.close(); - taxoDir.close(); - } - - @Test - public void testTwoNonEmptyByteArrays() throws Exception { - // add document with a category containing data for both enhancements - Directory indexDir = newDirectory(); - Directory taxoDir = newDirectory(); - - EnhancementsIndexingParams indexingParams = new EnhancementsIndexingParams( - new CategoryEnhancementDummy2(), new CategoryEnhancementDummy3()); - - List categoryPaths = new ArrayList(); - categoryPaths.add(new CategoryPath("a", "b")); - - RandomIndexWriter indexWriter = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))); - TaxonomyWriter taxo = new DirectoryTaxonomyWriter(taxoDir); - - // a category document builder will add the categories to a document - // once build() is called - Document doc = new Document(); - indexWriter.addDocument(new EnhancementsDocumentBuilder(taxo, - indexingParams).setCategoryPaths(categoryPaths).build(doc)); - - IndexReader indexReader = indexWriter.getReader(); - indexWriter.close(); - - Term term = DrillDown.term(indexingParams, new CategoryPath("a","b")); - EnhancementsPayloadIterator iterator = new EnhancementsPayloadIterator( - indexingParams.getCategoryEnhancements(), indexReader, term); - - assertTrue("EnhancementsPayloadIterator failure", iterator.init()); - assertTrue("Missing document 0", iterator.setdoc(0)); - byte[] dummy2 = (byte[]) iterator - .getCategoryData(new CategoryEnhancementDummy2()); - assertTrue("Bad array returned for CategoryEnhancementDummy2", Arrays - .equals(dummy2, CategoryEnhancementDummy2.CATEGORY_TOKEN_BYTES)); - byte[] dummy3 = (byte[]) iterator - .getCategoryData(new CategoryEnhancementDummy3()); - assertTrue("Bad array returned for CategoryEnhancementDummy3", Arrays - .equals(dummy3, CategoryEnhancementDummy3.CATEGORY_TOKEN_BYTES)); - indexReader.close(); - taxo.close(); - indexDir.close(); - taxoDir.close(); - } -} Index: lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/AssociationPropertyTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/AssociationPropertyTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/AssociationPropertyTest.java (working copy) @@ -1,65 +0,0 @@ -package org.apache.lucene.facet.enhancements.association; - -import org.junit.Test; - -import org.apache.lucene.facet.FacetException; -import org.apache.lucene.facet.enhancements.association.AssociationFloatProperty; -import org.apache.lucene.facet.enhancements.association.AssociationIntProperty; -import org.apache.lucene.facet.enhancements.association.AssociationProperty; -import org.apache.lucene.util.LuceneTestCase; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** Test {@link AssociationProperty}-ies. */ -public class AssociationPropertyTest extends LuceneTestCase { - - @Test - public void testAssociationCountProperty() throws FacetException { - AssociationProperty aa1 = new AssociationIntProperty(5); - AssociationProperty aa2 = new AssociationIntProperty(3); - assertEquals("Wrong association for property", 5, aa1.getAssociation()); - assertEquals("Wrong association for property", 3, aa2.getAssociation()); - aa1.merge(aa2); - assertEquals("Wrong association for property", 8, aa1.getAssociation()); - } - - @Test - public void testAssociationFloatProperty() throws FacetException { - AssociationFloatProperty aa1 = new AssociationFloatProperty(5); - AssociationFloatProperty aa2 = new AssociationFloatProperty(3); - assertEquals("Wrong association for property", 5.0, aa1.getFloatAssociation(), 0.00001); - assertEquals("Wrong association for property", 3.0, aa2.getFloatAssociation(), 0.00001); - aa1.merge(aa2); - assertEquals("Wrong association for property", 8.0, aa1.getFloatAssociation(), 0.00001); - } - - @Test - public void testEquals() { - AssociationProperty aa1 = new AssociationIntProperty(5); - AssociationProperty aa2 = new AssociationIntProperty(5); - AssociationProperty aa3 = new AssociationFloatProperty(5); - AssociationProperty aa4 = new AssociationFloatProperty(5); - - assertTrue("Should be equal", aa1.equals(aa1)); - assertTrue("Should be equal", aa1.equals(aa2)); - assertFalse("Should not be equal", aa1.equals(aa3)); - assertTrue("Should be equal", aa3.equals(aa3)); - assertTrue("Should be equal", aa3.equals(aa4)); - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/enhancements/association/CustomAssociationPropertyTest.java (working copy) @@ -1,99 +0,0 @@ -package org.apache.lucene.facet.enhancements.association; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.document.Document; -import org.apache.lucene.facet.enhancements.EnhancementsDocumentBuilder; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.CategoryContainer; -import org.apache.lucene.facet.index.attributes.CategoryAttributeImpl; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; -import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; -import org.junit.Test; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CustomAssociationPropertyTest extends LuceneTestCase { - - @Test - public void testCustomProperty() throws Exception { - class CustomProperty extends AssociationIntProperty { - public CustomProperty(int value) { - super(value); - } - @Override - public void merge(CategoryProperty other) { - throw new UnsupportedOperationException(); - } - } - - final int NUM_CATEGORIES = 10; - EnhancementsIndexingParams iParams = new EnhancementsIndexingParams(new AssociationEnhancement()); - - Directory iDir = newDirectory(); - Directory tDir = newDirectory(); - - RandomIndexWriter w = new RandomIndexWriter(random(), iDir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false))); - DirectoryTaxonomyWriter taxoW = new DirectoryTaxonomyWriter(tDir); - - CategoryContainer cc = new CategoryContainer(); - EnhancementsDocumentBuilder builder = new EnhancementsDocumentBuilder(taxoW, iParams); - for (int i = 1; i <= NUM_CATEGORIES; i++) { - CategoryAttributeImpl ca = new CategoryAttributeImpl(new CategoryPath(Integer.toString(i))); - ca.addProperty(new CustomProperty(i)); - - cc.addCategory(ca); - } - builder.setCategories(cc); - w.addDocument(builder.build(new Document())); - taxoW.close(); - IndexReader reader = w.getReader(); - w.close(); - - DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(tDir); - String field = iParams.getCategoryListParams(new CategoryPath("0")).getTerm().field(); - AssociationsPayloadIterator api = new AssociationsPayloadIterator(reader, field); - - api.setNextDoc(0); - - boolean flag = false; - for (int i = 1; i <= NUM_CATEGORIES; i++) { - int ordinal = taxo.getOrdinal(new CategoryPath(Integer.toString(i))); - flag = true; - long association = api.getAssociation(ordinal); - assertTrue("Association expected for ordinal "+ordinal+" but none was found", - association <= Integer.MAX_VALUE); - - assertEquals("Wrong association value for category '"+ i+"'", i, (int)association); - } - - assertTrue("No categories found for doc #0", flag); - - reader.close(); - taxo.close(); - iDir.close(); - tDir.close(); - } -} Index: lucene/facet/src/test/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParamsTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParamsTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/enhancements/params/EnhancementsIndexingParamsTest.java (working copy) @@ -1,51 +0,0 @@ -package org.apache.lucene.facet.enhancements.params; - -import java.util.List; - -import org.apache.lucene.facet.enhancements.CategoryEnhancement; -import org.apache.lucene.facet.enhancements.CategoryEnhancementDummy1; -import org.apache.lucene.facet.enhancements.CategoryEnhancementDummy2; -import org.apache.lucene.facet.index.DummyProperty; -import org.apache.lucene.facet.index.attributes.CategoryProperty; -import org.apache.lucene.util.LuceneTestCase; -import org.junit.Test; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class EnhancementsIndexingParamsTest extends LuceneTestCase { - - @Test - public void testCategoryEnhancements() { - EnhancementsIndexingParams params = new EnhancementsIndexingParams( - new CategoryEnhancementDummy1(), new CategoryEnhancementDummy2()); - - List enhancements = params.getCategoryEnhancements(); - assertEquals("Wrong number of enhancements", 2, enhancements.size()); - - // check order - assertTrue("Wrong first enhancement", enhancements.get(0) instanceof CategoryEnhancementDummy1); - assertTrue("Wrong second enhancement", enhancements.get(1) instanceof CategoryEnhancementDummy2); - - // check retainable properties - List retainableProps = params.getRetainableProperties(); - assertNotNull("Unexpected empty retainable list", retainableProps); - assertEquals("Unexpected size of retainable list", 1, retainableProps.size()); - assertSame("Wrong property in retainable list", DummyProperty.INSTANCE, retainableProps.get(0)); - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java (working copy) @@ -34,7 +34,7 @@ public class TestSimpleExample extends LuceneTestCase { @Test - public void testSimple () throws Exception { + public void testSimple() throws Exception { ExampleResult res = new SimpleMain().runSimple(); assertNotNull("Null result!", res); assertNotNull("Null facet result!", res.getFacetResults()); @@ -47,7 +47,7 @@ * As result, facets that without drill down got count of 2 will now get a count of 1. */ @Test - public void testDrillDown () throws Exception { + public void testDrillDown() throws Exception { ExampleResult res = new SimpleMain().runDrillDown(); assertNotNull("Null result!", res); assertNotNull("Null facet result!", res.getFacetResults()); Index: lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTest.java (working copy) @@ -1,236 +0,0 @@ -package org.apache.lucene.facet.index; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.util.Iterator; - -import org.junit.Test; - -import org.apache.lucene.facet.FacetException; -import org.apache.lucene.facet.enhancements.association.AssociationIntProperty; -import org.apache.lucene.facet.enhancements.association.AssociationProperty; -import org.apache.lucene.facet.index.CategoryContainer; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryAttributeImpl; -import org.apache.lucene.facet.index.streaming.CategoryAttributesStream; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryContainerTest extends CategoryContainerTestBase { - - @Test - public void basicTest() { - assertEquals("Wrong number of categories in the container", 3, - categoryContainer.size()); - - categoryContainer.clear(); - assertEquals("Container should not contain categories after clear", 0, - categoryContainer.size()); - } - - @Test - public void testIterator() throws FacetException { - Iterator iterator = categoryContainer.iterator(); - - // count the number of tokens - int nCategories; - for (nCategories = 0; iterator.hasNext(); nCategories++) { - iterator.next(); - } - assertEquals("Wrong number of tokens", 3, nCategories); - } - - @Test - public void testExistingNewCategoryWithProperty() throws FacetException { - categoryContainer.addCategory(new CategoryPath("five", "six"), - DummyProperty.INSTANCE); - Iterator iterator = categoryContainer.iterator(); - - // count the number of tokens, and check there is one DummyAttribute - int nCategories; - int nProperties = 0; - for (nCategories = 0; iterator.hasNext(); nCategories++) { - CategoryAttribute attribute = iterator.next(); - if (attribute.getProperty(DummyProperty.class) != null) { - nProperties++; - } - } - assertEquals("Wrong number of tokens", 3, nCategories); - assertEquals("Wrong number of tokens with properties", 1, nProperties); - } - - @Test - public void testMultipleCategoriesWithProperties() throws FacetException { - AssociationProperty associationProperty = new AssociationIntProperty( - 49); - categoryContainer.addCategory(new CategoryPath("five", "six"), - DummyProperty.INSTANCE, associationProperty); - categoryContainer.addCategory(new CategoryPath("seven", "eight"), - DummyProperty.INSTANCE); - associationProperty = new AssociationIntProperty(123); - categoryContainer.addCategory(new CategoryPath("nine"), - associationProperty, DummyProperty.INSTANCE); - Iterator iterator = categoryContainer.iterator(); - - // count the number of tokens, and check there is one DummyAttribute - int nCategories; - int nDummyAttributes = 0; - int nAssocAttributes = 0; - for (nCategories = 0; iterator.hasNext(); nCategories++) { - CategoryAttribute attribute = iterator.next(); - if (attribute.getProperty(DummyProperty.class) != null) { - nDummyAttributes++; - } - if (attribute.getProperty(AssociationIntProperty.class) != null) { - nAssocAttributes++; - } - } - assertEquals("Wrong number of tokens", 5, nCategories); - assertEquals("Wrong number of tokens with dummy properties", 3, - nDummyAttributes); - assertEquals("Wrong number of tokens with association properties", 2, - nAssocAttributes); - } - - @Test - public void testAddNewCategoryWithProperty() throws FacetException { - categoryContainer.addCategory(new CategoryPath("seven", "eight"), - DummyProperty.INSTANCE); - Iterator iterator = categoryContainer.iterator(); - - // count the number of tokens, and check there is one DummyAttribute - int nCategories; - int nProperties = 0; - for (nCategories = 0; iterator.hasNext(); nCategories++) { - CategoryAttribute attribute = iterator.next(); - if (attribute.getProperty(DummyProperty.class) != null) { - nProperties++; - } - } - assertEquals("Wrong number of tokens", 4, nCategories); - assertEquals("Wrong number of tokens with properties", 1, nProperties); - } - - /** - * Test addition of {@link CategoryAttribute} object without properties to a - * {@link CategoryContainer}. - */ - @Test - public void testAddCategoryAttributeWithoutProperties() - throws FacetException { - CategoryAttribute newCA = new CategoryAttributeImpl(new CategoryPath( - "seven", "eight")); - categoryContainer.addCategory(newCA); - } - - /** - * Test addition of {@link CategoryAttribute} object with property to a - * {@link CategoryContainer}. - */ - @Test - public void testAddCategoryAttributeWithProperty() throws FacetException { - CategoryAttribute newCA = new CategoryAttributeImpl(new CategoryPath( - "seven", "eight")); - newCA.addProperty(DummyProperty.INSTANCE); - categoryContainer.addCategory(newCA); - Iterator iterator = categoryContainer.iterator(); - - // count the number of tokens, and check there is one DummyAttribute - int nCategories; - int nProperties = 0; - for (nCategories = 0; iterator.hasNext(); nCategories++) { - CategoryAttribute attribute = iterator.next(); - if (attribute.getProperty(DummyProperty.class) != null) { - nProperties++; - } - } - assertEquals("Wrong number of tokens", 4, nCategories); - assertEquals("Wrong number of tokens with properties", 1, nProperties); - } - - /** - * Verifies that a {@link CategoryAttributesStream} can be constructed from - * {@link CategoryContainer} and produce the correct number of tokens. - */ - @Test - public void testCategoryAttributesStream() throws IOException { - CategoryAttributesStream stream = new CategoryAttributesStream( - categoryContainer); - // count the number of tokens - int nTokens; - for (nTokens = 0; stream.incrementToken(); nTokens++) { - } - assertEquals("Wrong number of tokens", 3, nTokens); - } - - /** - * Test that {@link CategoryContainer} merges properties. - */ - @Test - public void testCategoryAttributeMerge() throws FacetException { - categoryContainer.addCategory(initialCatgeories[0], - new AssociationIntProperty(2)); - categoryContainer.addCategory(initialCatgeories[0], - new AssociationIntProperty(15)); - - Iterator iterator = categoryContainer.iterator(); - - int nCategories; - int nAssociations = 0; - for (nCategories = 0; iterator.hasNext(); nCategories++) { - CategoryAttribute ca = iterator.next(); - AssociationProperty aa = (AssociationProperty) ca - .getProperty(AssociationIntProperty.class); - if (aa != null) { - assertEquals("Wrong association value", 17, aa.getAssociation()); - nAssociations++; - } - } - assertEquals("Wrong number of tokens", 3, nCategories); - assertEquals("Wrong number of tokens with associations", 1, - nAssociations); - } - - @Test - public void testSerialization() throws Exception { - AssociationProperty associationProperty = new AssociationIntProperty( - 49); - categoryContainer.addCategory(new CategoryPath("five", "six"), - DummyProperty.INSTANCE, associationProperty); - categoryContainer.addCategory(new CategoryPath("seven", "eight"), - DummyProperty.INSTANCE); - associationProperty = new AssociationIntProperty(123); - categoryContainer.addCategory(new CategoryPath("nine"), - associationProperty, DummyProperty.INSTANCE); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(1024); - ObjectOutputStream out = new ObjectOutputStream(baos); - out.writeObject(categoryContainer); - out.close(); - - ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); - ObjectInputStream in = new ObjectInputStream(bais); - assertEquals( - "Original and deserialized CategoryContainer are different", - categoryContainer, in.readObject()); - } -} \ No newline at end of file Index: lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTestBase.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTestBase.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/CategoryContainerTestBase.java (working copy) @@ -1,45 +0,0 @@ -package org.apache.lucene.facet.index; - -import org.junit.Before; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.facet.index.CategoryContainer; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public abstract class CategoryContainerTestBase extends LuceneTestCase { - - protected CategoryContainer categoryContainer; - protected CategoryPath[] initialCatgeories; - - @Before - public void setCategoryContainer() { - initialCatgeories = new CategoryPath[3]; - initialCatgeories[0] = new CategoryPath("one", "two", "three"); - initialCatgeories[1] = new CategoryPath("four"); - initialCatgeories[2] = new CategoryPath("five", "six"); - - categoryContainer = new CategoryContainer(); - - for (int i = 0; i < initialCatgeories.length; i++) { - categoryContainer.addCategory(initialCatgeories[i]); - } - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/index/CategoryListPayloadStreamTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/CategoryListPayloadStreamTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/CategoryListPayloadStreamTest.java (working copy) @@ -1,80 +0,0 @@ -package org.apache.lucene.facet.index; - -import java.io.ByteArrayInputStream; - -import org.junit.Test; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.facet.index.CategoryListPayloadStream; -import org.apache.lucene.util.encoding.DGapIntDecoder; -import org.apache.lucene.util.encoding.DGapIntEncoder; -import org.apache.lucene.util.encoding.IntDecoder; -import org.apache.lucene.util.encoding.NOnesIntDecoder; -import org.apache.lucene.util.encoding.NOnesIntEncoder; -import org.apache.lucene.util.encoding.UniqueValuesIntEncoder; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryListPayloadStreamTest extends LuceneTestCase { - - /** - * Verifies that a CategoryListPayloadStream can properly encode values into - * a byte stream for later constructing a Payload. - */ - @Test - public void testStream() throws Exception { - - CategoryListPayloadStream clps = new CategoryListPayloadStream( - new UniqueValuesIntEncoder(new DGapIntEncoder( - new NOnesIntEncoder(3)))); - - clps.appendIntToStream(1); - clps.appendIntToStream(10); - clps.appendIntToStream(100); - clps.appendIntToStream(1000); - clps.appendIntToStream(10000); - clps.appendIntToStream(100000); - clps.appendIntToStream(1000000); - clps.appendIntToStream(10000000); - clps.appendIntToStream(100000000); - clps.appendIntToStream(1000000000); - clps.appendIntToStream(Integer.MAX_VALUE); - - ByteArrayInputStream bais = new ByteArrayInputStream(clps - .convertStreamToByteArray()); - IntDecoder decoder = new DGapIntDecoder(new NOnesIntDecoder(3)); - decoder.reInit(bais); - assertEquals("Wrong value in byte stream", 1, decoder.decode()); - assertEquals("Wrong value in byte stream", 10, decoder.decode()); - assertEquals("Wrong value in byte stream", 100, decoder.decode()); - assertEquals("Wrong value in byte stream", 1000, decoder.decode()); - assertEquals("Wrong value in byte stream", 10000, decoder.decode()); - assertEquals("Wrong value in byte stream", 100000, decoder.decode()); - assertEquals("Wrong value in byte stream", 1000000, decoder.decode()); - assertEquals("Wrong value in byte stream", 10000000, decoder.decode()); - assertEquals("Wrong value in byte stream", 100000000, decoder.decode()); - assertEquals("Wrong value in byte stream", 1000000000, decoder.decode()); - assertEquals("Wrong value in byte stream", Integer.MAX_VALUE, decoder.decode()); - assertEquals("End of stream not reached", IntDecoder.EOS, decoder.decode()); - - clps.reset(); - decoder.reInit(bais); - assertEquals("End of stream not reached", IntDecoder.EOS, decoder.decode()); - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/index/DummyProperty.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/DummyProperty.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/DummyProperty.java (working copy) @@ -1,55 +0,0 @@ -package org.apache.lucene.facet.index; - -import org.apache.lucene.facet.index.attributes.CategoryProperty; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * An empty attribute for testing. - */ -public class DummyProperty implements CategoryProperty { - - public static final DummyProperty INSTANCE = new DummyProperty(); - - private DummyProperty() {} - - @Override - public boolean equals(Object o) { - if (o instanceof DummyProperty) { - return true; - } - return false; - } - - @Override - public int hashCode() { - return super.hashCode(); - } - - @Override - public void merge(CategoryProperty other) { - throw new UnsupportedOperationException( - "Merging dummy attribute is prohibited"); - } - - @Override - public String toString() { - return "I am dummy property"; - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java (working copy) @@ -101,9 +101,8 @@ int facetValue = asc? j: NUM_DOCS - j; categoryPaths.add(new CategoryPath("tag", Integer.toString(facetValue))); } - CategoryDocumentBuilder catBuilder = new CategoryDocumentBuilder(taxonomyWriter); - catBuilder.setCategoryPaths(categoryPaths); - catBuilder.build(doc); + FacetFields facetFields = new FacetFields(taxonomyWriter); + facetFields.addFields(doc, categoryPaths); writer.addDocument(doc); } taxonomyWriter.close(); Index: lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributeImplTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributeImplTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributeImplTest.java (working copy) @@ -1,125 +0,0 @@ -package org.apache.lucene.facet.index.attributes; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.Test; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.facet.FacetException; -import org.apache.lucene.facet.index.DummyProperty; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryAttributeImpl; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryAttributeImplTest extends LuceneTestCase { - - @Test - public void testCategoryPath() { - CategoryAttribute ca = new CategoryAttributeImpl(); - - assertNull("Category Path should be null", ca.getCategoryPath()); - - CategoryPath cp = new CategoryPath("a", "b"); - ca.setCategoryPath(cp); - - assertEquals("Wrong Category Path", cp, ca.getCategoryPath()); - - ca.setCategoryPath(null); - assertNull("Category Path should be null", ca.getCategoryPath()); - - ca = new CategoryAttributeImpl(cp); - assertEquals("Wrong Category Path", cp, ca.getCategoryPath()); - } - - @Test - public void testProperties() throws FacetException { - CategoryAttribute ca = new CategoryAttributeImpl(); - - assertNull("Attribute should be null", ca - .getProperty(DummyProperty.class)); - assertNull("Attribute classes should be null", ca.getPropertyClasses()); - - ca.addProperty(DummyProperty.INSTANCE); - assertEquals("DummyProperty should be in properties", - DummyProperty.INSTANCE, ca.getProperty(DummyProperty.class)); - assertEquals("Attribute classes should contain 1 element", 1, ca - .getPropertyClasses().size()); - - boolean failed = false; - try { - ca.addProperty(DummyProperty.INSTANCE); - } catch (UnsupportedOperationException e) { - failed = true; - } - - if (!failed) { - fail("Two DummyAttributes added to the same CategoryAttribute"); - } - - ca.clearProperties(); - assertNull("Attribute classes should be null", ca.getPropertyClasses()); - - ca.addProperty(DummyProperty.INSTANCE); - assertEquals("DummyProperty should be in properties", - DummyProperty.INSTANCE, ca.getProperty(DummyProperty.class)); - ca.remove(DummyProperty.class); - assertEquals("DummyProperty should not be in properties", null, ca - .getProperty(DummyProperty.class)); - assertNull("Attribute classes should be null", ca.getPropertyClasses()); - - ca.addProperty(DummyProperty.INSTANCE); - List> propertyClasses = new ArrayList>(); - assertEquals("No property expected when no classes given", null, ca - .getProperty(propertyClasses)); - propertyClasses.add(DummyProperty.class); - assertEquals("DummyProperty should be in properties", - DummyProperty.INSTANCE, ca.getProperty(propertyClasses)); - propertyClasses.add(OrdinalProperty.class); - assertEquals("DummyProperty should be in properties", - DummyProperty.INSTANCE, ca.getProperty(propertyClasses)); - propertyClasses.clear(); - propertyClasses.add(OrdinalProperty.class); - assertEquals("No ordinal property expected", null, ca - .getProperty(propertyClasses)); - } - - @Test - public void testCloneCopyToAndSet() throws FacetException { - CategoryAttributeImpl ca1 = new CategoryAttributeImpl(); - - CategoryPath cp = new CategoryPath("a", "b"); - ca1.setCategoryPath(cp); - ca1.addProperty(DummyProperty.INSTANCE); - - CategoryAttribute ca2 = ca1.clone(); - assertEquals("Error in cloning", ca1, ca2); - - CategoryAttributeImpl ca3 = new CategoryAttributeImpl(); - assertNotSame("Should not be the same", ca1, ca3); - ca1.copyTo(ca3); - assertEquals("Error in cloning", ca1, ca3); - - ca2.setCategoryPath(null); - assertNotSame("Should not be the same", ca1, ca2); - ca2.set(ca3); - assertEquals("Error in cloning", ca1, ca2); - } -} Index: lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributesIterableTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributesIterableTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/attributes/CategoryAttributesIterableTest.java (working copy) @@ -1,53 +0,0 @@ -package org.apache.lucene.facet.index.attributes; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.junit.Test; - -import org.apache.lucene.facet.index.CategoryContainerTestBase; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryAttributesIterable; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryAttributesIterableTest extends CategoryContainerTestBase { - - @Test - public void testIterator() throws IOException { - List categoryList = new ArrayList(); - for (int i = 0; i < initialCatgeories.length; i++) { - categoryList.add(initialCatgeories[i]); - } - - CategoryAttributesIterable iterable = new CategoryAttributesIterable( - categoryList); - Iterator iterator = iterable.iterator(); - - // count the number of tokens - int nCategories; - for (nCategories = 0; iterator.hasNext(); nCategories++) { - iterator.next(); - } - assertEquals("Wrong number of tokens", 3, nCategories); - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryAttributesStreamTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryAttributesStreamTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryAttributesStreamTest.java (working copy) @@ -1,80 +0,0 @@ -package org.apache.lucene.facet.index.streaming; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; - -import org.junit.Test; - -import org.apache.lucene.facet.index.CategoryContainerTestBase; -import org.apache.lucene.facet.index.attributes.CategoryAttribute; -import org.apache.lucene.facet.index.attributes.CategoryAttributeImpl; -import org.apache.lucene.facet.index.streaming.CategoryAttributesStream; -import org.apache.lucene.facet.taxonomy.CategoryPath; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryAttributesStreamTest extends CategoryContainerTestBase { - - /** - * Verifies that a {@link CategoryAttributesStream} accepts - * {@link CategoryAttribute} and passes them on as tokens. - */ - @Test - public void testStream() throws IOException { - ArrayList attributesList = new ArrayList(); - for (int i = 0; i < initialCatgeories.length; i++) { - attributesList.add(new CategoryAttributeImpl(initialCatgeories[i])); - } - - // test number of tokens - CategoryAttributesStream stream = new CategoryAttributesStream( - attributesList); - int nTokens = 0; - while (stream.incrementToken()) { - nTokens++; - } - assertEquals("Wrong number of tokens", 3, nTokens); - - // test reset - stream.reset(); - nTokens = 0; - while (stream.incrementToken()) { - nTokens++; - } - assertEquals("Wrong number of tokens", 3, nTokens); - - // test reset and contents - Set pathsSet = new HashSet(); - for (int i = 0; i < initialCatgeories.length; i++) { - pathsSet.add(initialCatgeories[i]); - } - stream.reset(); - while (stream.incrementToken()) { - CategoryAttribute fromStream = stream - .getAttribute(CategoryAttribute.class); - if (!pathsSet.remove(fromStream.getCategoryPath())) { - fail("Unexpected category path: " - + fromStream.getCategoryPath().toString(':')); - } - } - assertTrue("all category paths should have been found", pathsSet - .isEmpty()); - } -} Index: lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryParentsStreamTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryParentsStreamTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryParentsStreamTest.java (working copy) @@ -1,194 +0,0 @@ -package org.apache.lucene.facet.index.streaming; - -import java.io.IOException; - -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.facet.index.CategoryContainerTestBase; -import org.apache.lucene.facet.index.DummyProperty; -import org.apache.lucene.facet.index.categorypolicy.NonTopLevelOrdinalPolicy; -import org.apache.lucene.facet.index.categorypolicy.NonTopLevelPathPolicy; -import org.apache.lucene.facet.index.categorypolicy.OrdinalPolicy; -import org.apache.lucene.facet.index.categorypolicy.PathPolicy; -import org.apache.lucene.facet.index.params.FacetIndexingParams; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -import org.apache.lucene.store.Directory; -import org.junit.Test; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryParentsStreamTest extends CategoryContainerTestBase { - - /** - * Verifies that a {@link CategoryParentsStream} can be constructed from - * {@link CategoryAttributesStream} and produces the correct number of - * tokens with default facet indexing params. - */ - @Test - public void testStreamDefaultParams() throws IOException { - Directory directory = newDirectory(); - TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter( - directory); - CategoryParentsStream stream = new CategoryParentsStream( - new CategoryAttributesStream(categoryContainer), - taxonomyWriter, FacetIndexingParams.ALL_PARENTS); - - // count the number of tokens - int nTokens; - for (nTokens = 0; stream.incrementToken(); nTokens++) { - } - // should be 6 - all categories and parents - assertEquals("Wrong number of tokens", 6, nTokens); - - taxonomyWriter.close(); - directory.close(); - } - - /** - * Verifies that a {@link CategoryParentsStream} can be constructed from - * {@link CategoryAttributesStream} and produces the correct number of - * tokens with non top level facet indexing params. - */ - @Test - public void testStreamNonTopLevelParams() throws IOException { - Directory directory = newDirectory(); - final TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter( - directory); - FacetIndexingParams indexingParams = new FacetIndexingParams() { - @Override - public OrdinalPolicy getOrdinalPolicy() { - return new NonTopLevelOrdinalPolicy(); - } - @Override - public PathPolicy getPathPolicy() { - return new NonTopLevelPathPolicy(); - } - }; - - CategoryParentsStream stream = new CategoryParentsStream( - new CategoryAttributesStream(categoryContainer), - taxonomyWriter, indexingParams); - - // count the number of tokens - int nTokens; - for (nTokens = 0; stream.incrementToken(); nTokens++) { - } - /* - * should be 4: 3 non top level ("two", "three" and "six"), and one - * explicit top level ("four") - */ - assertEquals("Wrong number of tokens", 4, nTokens); - - taxonomyWriter.close(); - directory.close(); - } - - /** - * Verifies the correctness when no attributes in parents are retained in - * {@link CategoryParentsStream}. - */ - @Test - public void testNoRetainableAttributes() throws IOException { - Directory directory = newDirectory(); - TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(directory); - - assertNotNull(new CategoryParentsStream(new CategoryAttributesStream(categoryContainer), - taxonomyWriter, FacetIndexingParams.ALL_PARENTS)); - - // add DummyAttribute, but do not retain, only one expected - categoryContainer.addCategory(initialCatgeories[0], DummyProperty.INSTANCE); - - CategoryParentsStream stream = new CategoryParentsStream(new CategoryAttributesStream( - categoryContainer), taxonomyWriter, FacetIndexingParams.ALL_PARENTS); - - int nAttributes = 0; - while (stream.incrementToken()) { - if (stream.categoryAttribute.getProperty(DummyProperty.class) != null) { - nAttributes++; - } - } - assertEquals("Wrong number of tokens with attributes", 1, nAttributes); - - taxonomyWriter.close(); - directory.close(); - } - - /** - * Verifies the correctness when attributes in parents are retained in - * {@link CategoryParentsStream}. - */ - @Test - public void testRetainableAttributes() throws IOException { - Directory directory = newDirectory(); - TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(directory); - - FacetIndexingParams indexingParams = FacetIndexingParams.ALL_PARENTS; - assertNotNull(new CategoryParentsStream(new CategoryAttributesStream( - categoryContainer), taxonomyWriter, indexingParams)); - - // add DummyAttribute and retain it, three expected - categoryContainer.clear(); - categoryContainer.addCategory(initialCatgeories[0], DummyProperty.INSTANCE); - CategoryParentsStream stream = new CategoryParentsStream( - new CategoryAttributesStream(categoryContainer), - taxonomyWriter, FacetIndexingParams.ALL_PARENTS); - stream.addRetainableProperty(DummyProperty.class); - - MyCategoryListTokenizer tokenizer = new MyCategoryListTokenizer(stream, indexingParams); - - int nAttributes = 0; - try { - while (tokenizer.incrementToken()) { - if (stream.categoryAttribute.getProperty(DummyProperty.class) != null) { - nAttributes++; - } - } - } catch (IOException e) { - fail("Properties retained after stream closed"); - } - assertEquals("Wrong number of tokens with attributes", 3, nAttributes); - - taxonomyWriter.close(); - directory.close(); - } - - private final class MyCategoryListTokenizer extends CategoryListTokenizer { - - public MyCategoryListTokenizer(TokenStream input, - FacetIndexingParams indexingParams) { - super(input, indexingParams); - } - - @Override - public boolean incrementToken() throws IOException { - if (input.incrementToken()) { - return true; - } - if (categoryAttribute != null) { - if (categoryAttribute.getCategoryPath() == null) { - if (categoryAttribute.getProperty(DummyProperty.class) != null) { - throw new IOException( - "Properties not cleared properly from parents stream"); - } - } - } - return false; - } - - } -} \ No newline at end of file Index: lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryTokenizerTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryTokenizerTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/index/streaming/CategoryTokenizerTest.java (working copy) @@ -1,108 +0,0 @@ -package org.apache.lucene.facet.index.streaming; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.facet.index.CategoryContainerTestBase; -import org.apache.lucene.facet.index.attributes.CategoryAttributesIterable; -import org.apache.lucene.facet.index.params.FacetIndexingParams; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -import org.apache.lucene.store.Directory; -import org.junit.Test; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class CategoryTokenizerTest extends CategoryContainerTestBase { - - /** - * Verifies that a {@link CategoryTokenizer} adds the correct - * {@link CharTermAttribute}s to a {@link CategoryAttributesStream}. - */ - @Test - public void testTokensDefaultParams() throws IOException { - Directory directory = newDirectory(); - TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter( - directory); - FacetIndexingParams indexingParams = FacetIndexingParams.ALL_PARENTS; - CategoryTokenizer tokenizer = new CategoryTokenizer( - new CategoryAttributesStream(categoryContainer), - indexingParams); - - // count the number of tokens - Set categoryTerms = new HashSet(); - for (int i = 0; i < initialCatgeories.length; i++) { - categoryTerms.add(initialCatgeories[i] - .toString(indexingParams.getFacetDelimChar())); - } - - int nTokens; - for (nTokens = 0; tokenizer.incrementToken(); nTokens++) { - if (!categoryTerms.remove(tokenizer.termAttribute.toString())) { - fail("Unexpected term: " + tokenizer.termAttribute.toString()); - } - } - assertTrue("all category terms should have been found", categoryTerms - .isEmpty()); - - // should be 6 - all categories and parents - assertEquals("Wrong number of tokens", 3, nTokens); - - taxonomyWriter.close(); - directory.close(); - } - - /** - * Verifies that {@link CategoryTokenizer} elongates the buffer in - * {@link CharTermAttribute} for long categories. - */ - @Test - public void testLongCategoryPath() throws IOException { - Directory directory = newDirectory(); - TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter( - directory); - - List longCategory = new ArrayList(); - longCategory.add(new CategoryPath("one", "two", "three", "four", - "five", "six", "seven")); - - FacetIndexingParams indexingParams = FacetIndexingParams.ALL_PARENTS; - CategoryTokenizer tokenizer = new CategoryTokenizer( - new CategoryAttributesStream(new CategoryAttributesIterable( - longCategory)), indexingParams); - - // count the number of tokens - String categoryTerm = longCategory.get(0).toString( - indexingParams.getFacetDelimChar()); - - assertTrue("Missing token", tokenizer.incrementToken()); - if (!categoryTerm.equals(tokenizer.termAttribute.toString())) { - fail("Unexpected term: " + tokenizer.termAttribute.toString()); - } - - assertFalse("Unexpected token", tokenizer.incrementToken()); - - taxonomyWriter.close(); - directory.close(); - } -} Index: lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownTest.java (working copy) @@ -10,7 +10,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.index.params.PerDimensionIndexingParams; @@ -89,8 +89,8 @@ if (i % 5 == 0) { // 20 paths.add(new CategoryPath("b")); } - CategoryDocumentBuilder builder = new CategoryDocumentBuilder(taxoWriter); - builder.setCategoryPaths(paths).build(doc); + FacetFields facetFields = new FacetFields(taxoWriter); + facetFields.addFields(doc, paths); writer.addDocument(doc); } Index: lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java (working copy) @@ -22,7 +22,7 @@ import java.util.List; import org.apache.lucene.document.Document; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.search.params.CountFacetRequest; import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.search.results.FacetResult; @@ -43,7 +43,7 @@ private DirectoryTaxonomyWriter taxoWriter; private RandomIndexWriter writer; - private CategoryDocumentBuilder docBuilder; + private FacetFields docBuilder; private void add(String ... categoryPaths) throws IOException { Document doc = new Document(); @@ -52,8 +52,7 @@ for(String categoryPath : categoryPaths) { paths.add(new CategoryPath(categoryPath, '/')); } - docBuilder.setCategoryPaths(paths); - docBuilder.build(doc); + docBuilder.addFields(doc, paths); writer.addDocument(doc); } @@ -68,7 +67,7 @@ // Reused across documents, to add the necessary facet // fields: - docBuilder = new CategoryDocumentBuilder(taxoWriter); + docBuilder = new FacetFields(taxoWriter); add("Author/Bob", "Publish Date/2010/10/15"); add("Author/Lisa", "Publish Date/2010/10/20"); Index: lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java (working copy) @@ -1,13 +1,13 @@ package org.apache.lucene.facet.search; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.search.params.FacetSearchParams; import org.apache.lucene.facet.search.params.ScoreFacetRequest; import org.apache.lucene.facet.search.results.FacetResult; @@ -55,13 +55,12 @@ IndexWriter iw = new IndexWriter(indexDir, new IndexWriterConfig( TEST_VERSION_CURRENT, new KeywordAnalyzer())); - CategoryDocumentBuilder cdb = new CategoryDocumentBuilder(taxonomyWriter); - Iterable cats = Arrays.asList(new CategoryPath("a")); + FacetFields facetFields = new FacetFields(taxonomyWriter); for(int i = atLeast(2000); i > 0; --i) { Document doc = new Document(); doc.add(new StringField("f", "v", Store.NO)); - cdb.setCategoryPaths(cats); - iw.addDocument(cdb.build(doc)); + facetFields.addFields(doc, Collections.singletonList(new CategoryPath("a"))); + iw.addDocument(doc); } taxonomyWriter.close(); Index: lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java (working copy) @@ -2,13 +2,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.search.params.CountFacetRequest; import org.apache.lucene.facet.search.params.FacetRequest; @@ -321,11 +322,9 @@ private void prvt_add(FacetIndexingParams iParams, RandomIndexWriter iw, TaxonomyWriter tw, String... strings) throws IOException { - ArrayList cps = new ArrayList(); - CategoryPath cp = new CategoryPath(strings); - cps.add(cp); Document d = new Document(); - new CategoryDocumentBuilder(tw, iParams).setCategoryPaths(cps).build(d); + FacetFields facetFields = new FacetFields(tw, iParams); + facetFields.addFields(d, Collections.singletonList(new CategoryPath(strings))); d.add(new TextField("content", "alpha", Field.Store.YES)); iw.addDocument(d); } Index: lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java (working copy) @@ -2,7 +2,7 @@ import java.io.File; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; @@ -15,7 +15,7 @@ import org.apache.lucene.facet.example.TestMultiCLExample; import org.apache.lucene.facet.example.multiCL.MultiCLIndexer; import org.apache.lucene.facet.example.multiCL.MultiCLSearcher; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.search.TotalFacetCounts.CreationType; import org.apache.lucene.facet.search.results.FacetResult; @@ -88,10 +88,10 @@ /** Utility method to add a document and facets to an index/taxonomy. */ static void addFacets(FacetIndexingParams iParams, IndexWriter iw, TaxonomyWriter tw, String... strings) throws IOException { - ArrayList cps = new ArrayList(); - cps.add(new CategoryPath(strings)); - CategoryDocumentBuilder builder = new CategoryDocumentBuilder(tw, iParams); - iw.addDocument(builder.setCategoryPaths(cps).build(new Document())); + Document doc = new Document(); + FacetFields facetFields = new FacetFields(tw, iParams); + facetFields.addFields(doc, Collections.singletonList(new CategoryPath(strings))); + iw.addDocument(doc); } /** Clears the cache and sets its size to one. */ @@ -143,7 +143,6 @@ MockDirectoryWrapper indexDir = new MockDirectoryWrapper(random(), slowIndexDir); SlowRAMDirectory slowTaxoDir = new SlowRAMDirectory(-1, random()); MockDirectoryWrapper taxoDir = new MockDirectoryWrapper(random(), slowTaxoDir); - // Index documents without the "slowness" MultiCLIndexer.index(indexDir, taxoDir); Index: lucene/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/search/association/AssociationsFacetRequestTest.java (working copy) @@ -1,184 +0,0 @@ -package org.apache.lucene.facet.search.association; - -import java.util.List; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.document.Document; -import org.apache.lucene.facet.enhancements.EnhancementsDocumentBuilder; -import org.apache.lucene.facet.enhancements.association.AssociationEnhancement; -import org.apache.lucene.facet.enhancements.association.AssociationFloatProperty; -import org.apache.lucene.facet.enhancements.association.AssociationIntProperty; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.CategoryContainer; -import org.apache.lucene.facet.search.FacetsCollector; -import org.apache.lucene.facet.search.params.FacetSearchParams; -import org.apache.lucene.facet.search.params.association.AssociationFloatSumFacetRequest; -import org.apache.lucene.facet.search.params.association.AssociationIntSumFacetRequest; -import org.apache.lucene.facet.search.results.FacetResult; -import org.apache.lucene.facet.taxonomy.CategoryPath; -import org.apache.lucene.facet.taxonomy.TaxonomyWriter; -import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; -import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** Test for associations */ -public class AssociationsFacetRequestTest extends LuceneTestCase { - - private static Directory dir; - private static IndexReader reader; - private static Directory taxoDir; - - private static final CategoryPath aint = new CategoryPath("int", "a"); - private static final CategoryPath bint = new CategoryPath("int", "b"); - private static final CategoryPath afloat = new CategoryPath("float", "a"); - private static final CategoryPath bfloat = new CategoryPath("float", "b"); - - @BeforeClass - public static void beforeClassAssociationsFacetRequestTest() throws Exception { - dir = newDirectory(); - taxoDir = newDirectory(); - // preparations - index, taxonomy, content - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random(), MockTokenizer.KEYWORD, false))); - - TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir); - - EnhancementsDocumentBuilder builder = new EnhancementsDocumentBuilder( - taxoWriter, new EnhancementsIndexingParams(new AssociationEnhancement())); - - // index documents, 50% have only 'b' and all have 'a' - for (int i = 0; i < 100; i++) { - Document doc = new Document(); - CategoryContainer container = new CategoryContainer(); - container.addCategory(aint, new AssociationIntProperty(2)); - container.addCategory(afloat, new AssociationFloatProperty(0.5f)); - if (i % 2 == 0) { // 50 - container.addCategory(bint, new AssociationIntProperty(3)); - container.addCategory(bfloat, new AssociationFloatProperty(0.2f)); - } - builder.setCategories(container).build(doc); - writer.addDocument(doc); - } - - taxoWriter.close(); - reader = writer.getReader(); - writer.close(); - } - - @AfterClass - public static void afterClassAssociationsFacetRequestTest() throws Exception { - reader.close(); - reader = null; - dir.close(); - dir = null; - taxoDir.close(); - taxoDir = null; - } - - @Test - public void testIntSumAssociation() throws Exception { - DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); - - // facet requests for two facets - FacetSearchParams fsp = new FacetSearchParams( - new AssociationIntSumFacetRequest(aint, 10), - new AssociationIntSumFacetRequest(bint, 10)); - - Query q = new MatchAllDocsQuery(); - - FacetsCollector fc = new FacetsCollector(fsp, reader, taxo); - - IndexSearcher searcher = newSearcher(reader); - searcher.search(q, fc); - List res = fc.getFacetResults(); - - assertNotNull("No results!",res); - assertEquals("Wrong number of results!",2, res.size()); - assertEquals("Wrong count for category 'a'!",200, (int) res.get(0).getFacetResultNode().getValue()); - assertEquals("Wrong count for category 'b'!",150, (int) res.get(1).getFacetResultNode().getValue()); - - taxo.close(); - } - - @Test - public void testFloatSumAssociation() throws Exception { - DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); - - // facet requests for two facets - FacetSearchParams fsp = new FacetSearchParams( - new AssociationFloatSumFacetRequest(afloat, 10), - new AssociationFloatSumFacetRequest(bfloat, 10)); - - Query q = new MatchAllDocsQuery(); - - FacetsCollector fc = new FacetsCollector(fsp, reader, taxo); - - IndexSearcher searcher = newSearcher(reader); - searcher.search(q, fc); - List res = fc.getFacetResults(); - - assertNotNull("No results!",res); - assertEquals("Wrong number of results!",2, res.size()); - assertEquals("Wrong count for category 'a'!",50f, (float) res.get(0).getFacetResultNode().getValue(), 0.00001); - assertEquals("Wrong count for category 'b'!",10f, (float) res.get(1).getFacetResultNode().getValue(), 0.00001); - - taxo.close(); - } - - @Test - public void testDifferentAggregatorsSameCategoryList() throws Exception { - // Same category list cannot be aggregated by two different aggregators. If - // you want to do that, you need to separate the categories into two - // category list (you'll still have one association list). - DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); - - // facet requests for two facets - FacetSearchParams fsp = new FacetSearchParams( - new AssociationIntSumFacetRequest(aint, 10), - new AssociationIntSumFacetRequest(bint, 10), - new AssociationFloatSumFacetRequest(afloat, 10), - new AssociationFloatSumFacetRequest(bfloat, 10)); - - Query q = new MatchAllDocsQuery(); - - FacetsCollector fc = new FacetsCollector(fsp, reader, taxo); - - IndexSearcher searcher = newSearcher(reader); - searcher.search(q, fc); - try { - fc.getFacetResults(); - fail("different aggregators for same category list should not be supported"); - } catch (RuntimeException e) { - // ok - expected - } - taxo.close(); - } - -} Index: lucene/facet/src/test/org/apache/lucene/facet/search/associations/AssociationsFacetRequestTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/search/associations/AssociationsFacetRequestTest.java (working copy) +++ lucene/facet/src/test/org/apache/lucene/facet/search/associations/AssociationsFacetRequestTest.java (working copy) @@ -1,20 +1,18 @@ -package org.apache.lucene.facet.search.association; +package org.apache.lucene.facet.search.associations; import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.facet.enhancements.EnhancementsDocumentBuilder; -import org.apache.lucene.facet.enhancements.association.AssociationEnhancement; -import org.apache.lucene.facet.enhancements.association.AssociationFloatProperty; -import org.apache.lucene.facet.enhancements.association.AssociationIntProperty; -import org.apache.lucene.facet.enhancements.params.EnhancementsIndexingParams; -import org.apache.lucene.facet.index.CategoryContainer; +import org.apache.lucene.facet.associations.AssociationsFacetFields; +import org.apache.lucene.facet.associations.CategoryAssociationsContainer; +import org.apache.lucene.facet.associations.CategoryFloatAssociation; +import org.apache.lucene.facet.associations.CategoryIntAssociation; import org.apache.lucene.facet.search.FacetsCollector; import org.apache.lucene.facet.search.params.FacetSearchParams; -import org.apache.lucene.facet.search.params.association.AssociationFloatSumFacetRequest; -import org.apache.lucene.facet.search.params.association.AssociationIntSumFacetRequest; +import org.apache.lucene.facet.search.params.associations.AssociationFloatSumFacetRequest; +import org.apache.lucene.facet.search.params.associations.AssociationIntSumFacetRequest; import org.apache.lucene.facet.search.results.FacetResult; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; @@ -70,20 +68,19 @@ TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir); - EnhancementsDocumentBuilder builder = new EnhancementsDocumentBuilder( - taxoWriter, new EnhancementsIndexingParams(new AssociationEnhancement())); + AssociationsFacetFields assocFacetFields = new AssociationsFacetFields(taxoWriter); // index documents, 50% have only 'b' and all have 'a' for (int i = 0; i < 100; i++) { Document doc = new Document(); - CategoryContainer container = new CategoryContainer(); - container.addCategory(aint, new AssociationIntProperty(2)); - container.addCategory(afloat, new AssociationFloatProperty(0.5f)); + CategoryAssociationsContainer associations = new CategoryAssociationsContainer(); + associations.setAssociation(aint, new CategoryIntAssociation(2)); + associations.setAssociation(afloat, new CategoryFloatAssociation(0.5f)); if (i % 2 == 0) { // 50 - container.addCategory(bint, new AssociationIntProperty(3)); - container.addCategory(bfloat, new AssociationFloatProperty(0.2f)); + associations.setAssociation(bint, new CategoryIntAssociation(3)); + associations.setAssociation(bfloat, new CategoryFloatAssociation(0.2f)); } - builder.setCategories(container).build(doc); + assocFacetFields.addFields(doc, associations); writer.addDocument(doc); } Index: lucene/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/search/params/MultiIteratorsPerCLParamsTest.java (working copy) @@ -8,7 +8,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.index.params.CategoryListParams; import org.apache.lucene.facet.index.params.FacetIndexingParams; import org.apache.lucene.facet.search.CategoryListIterator; @@ -171,11 +171,11 @@ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false))); TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir); + FacetFields facetFields = new FacetFields(taxoWriter, iParams); for (CategoryPath[] categories : perDocCategories) { - writer.addDocument(new CategoryDocumentBuilder(taxoWriter, iParams) - .setCategoryPaths(Arrays.asList(categories)).build( - new Document())); - + Document doc = new Document(); + facetFields.addFields(doc, Arrays.asList(categories)); + writer.addDocument(doc); } taxoWriter.commit(); writer.commit(); Index: lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java (working copy) @@ -1,11 +1,11 @@ package org.apache.lucene.facet.search.sampling; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collections; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.search.FacetsAccumulator; import org.apache.lucene.facet.search.FacetsCollector; import org.apache.lucene.facet.search.params.CountFacetRequest; @@ -100,14 +100,12 @@ IndexWriter w = new IndexWriter(indexDir, iwc); TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir); - CategoryDocumentBuilder cdb = new CategoryDocumentBuilder(tw); - ArrayList categoryPaths = new ArrayList(1); - + FacetFields facetFields = new FacetFields(tw); for (int i = 0; i < 100; i++) { - categoryPaths.clear(); - categoryPaths.add(new CategoryPath("root",Integer.toString(i / 10), Integer.toString(i))); - cdb.setCategoryPaths(categoryPaths); - w.addDocument(cdb.build(new Document())); + Document doc = new Document(); + CategoryPath cp = new CategoryPath("root",Integer.toString(i / 10), Integer.toString(i)); + facetFields.addFields(doc, Collections.singletonList(cp)); + w.addDocument(doc); } IOUtils.close(tw, w); } Index: lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java (working copy) @@ -8,7 +8,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.document.Document; -import org.apache.lucene.facet.index.CategoryDocumentBuilder; +import org.apache.lucene.facet.index.FacetFields; import org.apache.lucene.facet.taxonomy.CategoryPath; import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache; import org.apache.lucene.facet.taxonomy.writercache.cl2o.Cl2oTaxonomyWriterCache; @@ -94,7 +94,7 @@ for (int i = 0; i < indexThreads.length; i++) { indexThreads[i] = new Thread() { - private final CategoryDocumentBuilder cdb = new CategoryDocumentBuilder(tw); + private final FacetFields facetFields = new FacetFields(tw); @Override public void run() { @@ -115,8 +115,7 @@ --level; } } - cdb.setCategoryPaths(cats); - cdb.build(doc); + facetFields.addFields(doc, cats); iw.addDocument(doc); } catch (IOException e) { throw new RuntimeException(e); Index: lucene/facet/src/test/org/apache/lucene/util/collections/IntHashSetTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/util/collections/IntHashSetTest.java (revision 1427505) +++ lucene/facet/src/test/org/apache/lucene/util/collections/IntHashSetTest.java (working copy) @@ -121,10 +121,8 @@ } } assertEquals(set2.size(), set1.size()); - int i = 0; for (int value : set2) { assertTrue(set1.contains(value)); - i++; } } Index: lucene/facet/src/test/org/apache/lucene/util/collections/IntToFloatMapTest.java =================================================================== --- lucene/facet/src/test/org/apache/lucene/util/collections/IntToFloatMapTest.java (revision 0) +++ lucene/facet/src/test/org/apache/lucene/util/collections/IntToFloatMapTest.java (working copy) @@ -0,0 +1,272 @@ +package org.apache.lucene.util.collections; + +import org.junit.Test; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.collections.FloatIterator; +import org.apache.lucene.util.collections.IntIterator; +import org.apache.lucene.util.collections.IntToFloatMap; + +import java.util.HashSet; +import java.util.Random; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class IntToFloatMapTest extends LuceneTestCase { + private static void assertGround(float value) { + assertEquals(IntToFloatMap.GROUND, value, Float.MAX_VALUE); + } + + @Test + public void test0() { + IntToFloatMap map = new IntToFloatMap(); + + assertGround(map.get(0)); + + for (int i = 0; i < 100; ++i) { + int value = 100 + i; + assertFalse(map.containsValue(value)); + map.put(i, value); + assertTrue(map.containsValue(value)); + assertNotNull(map.get(i)); + } + + assertEquals(100, map.size()); + for (int i = 0; i < 100; ++i) { + assertTrue(map.containsKey(i)); + assertEquals(100 + i, map.get(i), Float.MAX_VALUE); + + } + + for (int i = 10; i < 90; ++i) { + map.remove(i); + assertGround(map.get(i)); + } + + assertEquals(20, map.size()); + for (int i = 0; i < 100; ++i) { + assertEquals(map.containsKey(i), !(i >= 10 && i < 90)); + } + + for (int i = 5; i < 85; ++i) { + map.put(i, Integer.valueOf(5 + i)); + } + assertEquals(95, map.size()); + for (int i = 0; i < 100; ++i) { + assertEquals(map.containsKey(i), !(i >= 85 && i < 90)); + } + for (int i = 0; i < 5; ++i) { + assertEquals(map.get(i), (100 + i), Float.MAX_VALUE); + } + for (int i = 5; i < 85; ++i) { + assertEquals(map.get(i), (5 + i), Float.MAX_VALUE); + } + for (int i = 90; i < 100; ++i) { + assertEquals(map.get(i), (100 + i), Float.MAX_VALUE); + } + } + + @Test + public void test1() { + IntToFloatMap map = new IntToFloatMap(); + + for (int i = 0; i < 100; ++i) { + map.put(i, Integer.valueOf(100 + i)); + } + + HashSet set = new HashSet(); + + for (FloatIterator iterator = map.iterator(); iterator.hasNext();) { + set.add(iterator.next()); + } + + assertEquals(set.size(), map.size()); + for (int i = 0; i < 100; ++i) { + assertTrue(set.contains(Float.valueOf(100+i))); + } + + set.clear(); + for (FloatIterator iterator = map.iterator(); iterator.hasNext();) { + float d = iterator.next(); + if (d % 2 == 1) { + iterator.remove(); + continue; + } + set.add(d); + } + assertEquals(set.size(), map.size()); + for (int i = 0; i < 100; i+=2) { + assertTrue(set.contains(Float.valueOf(100+i))); + } + } + + @Test + public void test2() { + IntToFloatMap map = new IntToFloatMap(); + + assertTrue(map.isEmpty()); + assertGround(map.get(0)); + for (int i = 0; i < 128; ++i) { + int value = i * 4096; + assertFalse(map.containsValue(value)); + map.put(i, value); + assertTrue(map.containsValue(value)); + assertNotNull(map.get(i)); + assertFalse(map.isEmpty()); + } + + assertEquals(128, map.size()); + for (int i = 0; i < 128; ++i) { + assertTrue(map.containsKey(i)); + assertEquals(i * 4096, map.get(i), Float.MAX_VALUE); + } + + for (int i = 0 ; i < 200; i+=2) { + map.remove(i); + } + assertEquals(64, map.size()); + for (int i = 1; i < 128; i+=2) { + assertTrue(map.containsKey(i)); + assertEquals(i * 4096, map.get(i), Float.MAX_VALUE); + map.remove(i); + } + assertTrue(map.isEmpty()); + } + + @Test + public void test3() { + IntToFloatMap map = new IntToFloatMap(); + int length = 100; + for (int i = 0; i < length; ++i) { + map.put(i*64, 100 + i); + } + HashSet keySet = new HashSet(); + for (IntIterator iit = map.keyIterator(); iit.hasNext(); ) { + keySet.add(iit.next()); + } + assertEquals(length, keySet.size()); + for (int i = 0; i < length; ++i) { + assertTrue(keySet.contains(i * 64)); + } + + HashSet valueSet = new HashSet(); + for (FloatIterator iit = map.iterator(); iit.hasNext(); ) { + valueSet.add(iit.next()); + } + assertEquals(length, valueSet.size()); + float[] array = map.toArray(); + assertEquals(length, array.length); + for (float value: array) { + assertTrue(valueSet.contains(value)); + } + + float[] array2 = new float[80]; + array2 = map.toArray(array2); + assertEquals(length, array2.length); + for (float value: array2) { + assertTrue(valueSet.contains(value)); + } + + float[] array3 = new float[120]; + array3 = map.toArray(array3); + for (int i = 0 ;i < length; ++i) { + assertTrue(valueSet.contains(array3[i])); + } + + for (int i = 0; i < length; ++i) { + assertTrue(map.containsValue(i + 100)); + assertTrue(map.containsKey(i*64)); + } + + for (IntIterator iit = map.keyIterator(); iit.hasNext(); ) { + iit.next(); + iit.remove(); + } + assertTrue(map.isEmpty()); + assertEquals(0, map.size()); + + } + + // now with random data.. and lots of it + @Test + public void test4() { + IntToFloatMap map = new IntToFloatMap(); + int length = ArrayHashMapTest.RANDOM_TEST_NUM_ITERATIONS; + // for a repeatable random sequence + long seed = random().nextLong(); + Random random = new Random(seed); + + for (int i = 0; i < length; ++i) { + int value = random.nextInt(Integer.MAX_VALUE); + map.put(i*128, value); + } + + assertEquals(length, map.size()); + + // now repeat + random.setSeed(seed); + + for (int i = 0; i < length; ++i) { + int value = random.nextInt(Integer.MAX_VALUE); + assertTrue(map.containsValue(value)); + assertTrue(map.containsKey(i*128)); + assertEquals(0, Float.compare(value, map.remove(i*128))); + } + assertEquals(0, map.size()); + assertTrue(map.isEmpty()); + } + + @Test + public void testEquals() { + IntToFloatMap map1 = new IntToFloatMap(100); + IntToFloatMap map2 = new IntToFloatMap(100); + assertEquals("Empty maps should be equal", map1, map2); + assertEquals("hashCode() for empty maps should be equal", + map1.hashCode(), map2.hashCode()); + + for (int i = 0; i < 100; ++i) { + map1.put(i, Float.valueOf(1f/i)); + map2.put(i, Float.valueOf(1f/i)); + } + assertEquals("Identical maps should be equal", map1, map2); + assertEquals("hashCode() for identical maps should be equal", + map1.hashCode(), map2.hashCode()); + + for (int i = 10; i < 20; i++) { + map1.remove(i); + } + assertFalse("Different maps should not be equal", map1.equals(map2)); + + for (int i = 19; i >=10; --i) { + map2.remove(i); + } + assertEquals("Identical maps should be equal", map1, map2); + assertEquals("hashCode() for identical maps should be equal", + map1.hashCode(), map2.hashCode()); + + map1.put(-1,-1f); + map2.put(-1,-1.1f); + assertFalse("Different maps should not be equal", map1.equals(map2)); + + map2.put(-1,-1f); + assertEquals("Identical maps should be equal", map1, map2); + assertEquals("hashCode() for identical maps should be equal", + map1.hashCode(), map2.hashCode()); + } + +} Property changes on: lucene/facet/src/test/org/apache/lucene/util/collections/IntToFloatMapTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property