Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/SearcherTaxoManagerTest.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/search/SearcherTaxoManagerTest.java (revision 0) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/search/SearcherTaxoManagerTest.java (working copy) @@ -0,0 +1,88 @@ +package org.apache.lucene.facet.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.facet.search.SearcherTaxoManager.TaxonomyTimestamp; +import org.apache.lucene.facet.taxonomy.CategoryPath; +import org.apache.lucene.facet.taxonomy.TaxonomyReader; +import org.apache.lucene.facet.taxonomy.TaxonomyWriter; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LuceneTestCase; +import org.junit.Test; + +public class SearcherTaxoManagerTest extends LuceneTestCase { + + private void touchIndex(Directory searchDir, Directory taxoDir) throws IOException { + String timeString = Long.toString(System.currentTimeMillis()); + TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir); + tw.addCategory(new CategoryPath("a/"+Integer.toString(tw.getSize()), '/')); + Map commitData = new HashMap(); + commitData.put(TaxonomyTimestamp.INDEX_UPDATE_TIME, timeString); + commitData.put(TaxonomyTimestamp.INDEX_CREATE_TIME, timeString); + tw.commit(commitData); + tw.close(); + + IndexWriter writer = new IndexWriter(searchDir, new IndexWriterConfig( + TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + writer.addDocument(new Document()); + commitData.clear(); + commitData.put(TaxonomyTimestamp.TAXO_UPDATE_TIME, timeString); + commitData.put(TaxonomyTimestamp.TAXO_CREATE_TIME, timeString); + writer.commit(commitData); + writer.close(); + } + + private void touchIndexNonSynched(Directory dir, Directory taxoDir) throws IOException { + TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir); + tw.addCategory(new CategoryPath("a/"+Integer.toString(tw.getSize()), '/')); + tw.commit(); + tw.close(); + + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( + TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + writer.addDocument(new Document()); + long time = System.nanoTime(); + // TODO: read createTime from the taxoDir following the commit + writer.commit(new TaxonomyTimestamp(-1, time + 1000).toSearchCommitData()); + writer.close(); + } + + @Test(expected=IOException.class) + public void testCtorWithNonSynchedIndexes() throws Exception { + Directory dir = newDirectory(); + Directory taxoDir = newDirectory(); + touchIndexNonSynched(dir, taxoDir); + // assert just to avoid the "object not used" compile warning... + assertNotNull(new SearcherTaxoManager(dir, taxoDir, null)); + fail("should not have reached here - SearcherTaxoManager should fail if search and taxonomy indexes are out of sync"); + } + +} Property changes on: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/SearcherTaxoManagerTest.java ___________________________________________________________________ Added: svn:executable ## -0,0 +1 ## +* Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java =================================================================== --- lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java (revision 1326275) +++ lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java (working copy) @@ -200,5 +200,29 @@ dir.close(); } + + @Test + public void testTimestamps() throws Exception { + // ensures that an application cannot override the create/update time stamps. + Directory dir = newDirectory(); + + DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache()); + taxoWriter.addCategory(new CategoryPath("a")); + HashMap commitUserData = new HashMap(); + // set these properties -- they should not be committed like that ! + commitUserData.put(DirectoryTaxonomyWriter.INDEX_CREATE_TIME, "create"); + commitUserData.put(DirectoryTaxonomyWriter.INDEX_UPDATE_TIME, "update"); + taxoWriter.commit(commitUserData); + taxoWriter.close(); + + DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir); + Map commitData = taxoReader.getCommitUserData(); + taxoReader.close(); + dir.close(); + + // parse the timestamp properties -- they should be parseable (i.e. not overridden above). + Long.parseLong(commitData.get(DirectoryTaxonomyWriter.INDEX_CREATE_TIME)); + Long.parseLong(commitData.get(DirectoryTaxonomyWriter.INDEX_UPDATE_TIME)); + } } Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/SearcherTaxoManager.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/SearcherTaxoManager.java (revision 0) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/SearcherTaxoManager.java (working copy) @@ -0,0 +1,286 @@ +package org.apache.lucene.facet.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.lucene.facet.search.SearcherTaxoManager.SearcherTaxoPair; +import org.apache.lucene.facet.taxonomy.InconsistentTaxonomyException; +import org.apache.lucene.facet.taxonomy.TaxonomyReader; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; + +public class SearcherTaxoManager extends ReferenceManager { + + public static class SearcherTaxoPair { + public final IndexSearcher searcher; + public final TaxonomyReader taxoReader; + + public SearcherTaxoPair(IndexSearcher is, TaxonomyReader tr) { + searcher = is; + taxoReader = tr; + } + } + + private static final Logger logger = Logger.getLogger(SearcherTaxoManager.class.getName()); + + /** Holds the create and last update times of a taxonomy index. */ + public static final class TaxonomyTimestamp { + + /** + * The property name that exists in the commit data of the search index, and + * denotes the taxonomy index update time. + */ + public static final String TAXO_UPDATE_TIME = "TAXONOMY_INDEX_UPDATE_TIME"; + + /** + * The property name that exists in the commit data of the search index, and + * denotes the taxonomy index creation time. + */ + public static final String TAXO_CREATE_TIME = "TAXONOMY_INDEX_CREATE_TIME"; + + public final long createTime; + public final long lastUpdateTime; + + /** Creates a {@link TaxonomyTimestamp} from the commit data of a taxonomy index. */ + public static TaxonomyTimestamp fromTaxonomyCommitData(Map commitData) { + long createTime = -1, updateTime = -1; + if (commitData != null) { + String createTimeProp = commitData.get(DirectoryTaxonomyWriter.INDEX_CREATE_TIME); + if (createTimeProp != null) { + createTime = Long.parseLong(createTimeProp); + } + + String updateTimeProp = commitData.get(DirectoryTaxonomyWriter.INDEX_UPDATE_TIME); + if (updateTimeProp != null) { + updateTime = Long.parseLong(updateTimeProp); + } + } + return new TaxonomyTimestamp(createTime, updateTime); + } + + /** Creates a {@link TaxonomyTimestamp} from the commit data of a search index. */ + public static TaxonomyTimestamp fromSearchCommitData(Map commitData) { + long createTime = -1, updateTime = -1; + if (commitData != null) { + String createTimeProp = commitData.get(TAXO_CREATE_TIME); + if (createTimeProp != null) { + createTime = Long.parseLong(createTimeProp); + } + + String updateTimeProp = commitData.get(TAXO_UPDATE_TIME); + if (updateTimeProp != null) { + updateTime = Long.parseLong(updateTimeProp); + } + } + return new TaxonomyTimestamp(createTime, updateTime); + } + + public TaxonomyTimestamp(long createTime, long lastUpdateTime) { + this.createTime = createTime; + this.lastUpdateTime = lastUpdateTime; + } + + public Map toSearchCommitData() { + return new HashMap() {{ + put(TAXO_CREATE_TIME, Long.toString(createTime)); + put(TAXO_UPDATE_TIME, Long.toString(lastUpdateTime)); + }}; + } + + @Override + public String toString() { + return "createTime=" + createTime + " lastUpdateTime=" + lastUpdateTime; + } + + } + + private final SearcherFactory searcherFactory; + private final Directory taxoDir; + + public SearcherTaxoManager(Directory indexDir, Directory taxoDir, + SearcherFactory searcherFactory) throws IOException { + if (indexDir == null || taxoDir == null) { + throw new IllegalArgumentException("indexReader, taxoReader and taxoFactory cannot be null !"); + } + + IndexReader indexReader = null; + TaxonomyReader taxoReader = null; + boolean success = false; + try { + indexReader = IndexReader.open(taxoDir); + taxoReader = new DirectoryTaxonomyReader(taxoDir); + + // validate that the IndexReader and TaxoReader are in sync + TaxonomyTimestamp searchTaxoToken = getTimestampToken(indexReader); + TaxonomyTimestamp taxoIndexToken = getTimestampToken(taxoReader); + if (searchTaxoToken.createTime != taxoIndexToken.createTime + || searchTaxoToken.lastUpdateTime > taxoIndexToken.lastUpdateTime) { + String msg = "Search and Taxonomy indexes times do not match: searchIndex=[" + + searchTaxoToken + "]; taxoIndex=[" + taxoIndexToken + "]"; + if (logger.isLoggable(Level.FINEST)) { + logger.finest(msg); + } + throw new IOException(msg); + } + + if (searcherFactory == null) { + searcherFactory = new SearcherFactory(); + } + + this.searcherFactory = searcherFactory; + current = new SearcherTaxoPair(searcherFactory.newSearcher(indexReader), taxoReader); + this.taxoDir = taxoDir; + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(indexReader, taxoReader); + } + } + } + + @Override + protected void decRef(SearcherTaxoPair reference) throws IOException { + boolean success1 = false, success2 = false; + try { + reference.searcher.getIndexReader().decRef(); + success1 = true; + reference.taxoReader.decRef(); + success2 = true; + } finally { + if (!success1) { + reference.searcher.getIndexReader().incRef(); + } + if (!success2) { + reference.taxoReader.incRef(); + } + } + } + + @Override + protected boolean tryIncRef(SearcherTaxoPair reference) { + if (reference.searcher.getIndexReader().tryIncRef()) { + reference.taxoReader.incRef(); + return true; + } + return false; + } + + /** Returns the taxonomy index create/update timestamps that are registered on the search index. */ + private TaxonomyTimestamp getTimestampToken(IndexReader reader) throws IOException { + return TaxonomyTimestamp.fromSearchCommitData(reader.getIndexCommit().getUserData()); + } + + /** Returns the taxonomy index create/update timestamps that are registered on the taxonomy index. */ + private TaxonomyTimestamp getTimestampToken(TaxonomyReader taxoReader) throws IOException { + return TaxonomyTimestamp.fromTaxonomyCommitData(taxoReader.getCommitUserData()); + } + + /** + * Returns a new {@link SearcherTaxoPair} if the search and taxonomy index are + * in sync, otherwise returns {@code null}. + */ + private SearcherTaxoPair getNewPair(TaxonomyReader taxoReader, + IndexReader reader, TaxonomyTimestamp readerTaxoToken, + TaxonomyTimestamp taxoReaderToken) throws IOException { + SearcherTaxoPair pair = null; + try { + // search and taxonomy indexes are in the same epoch. Refresh taxoReader + // and return a new pair if appropriate + taxoReader.refresh(); + + // The taxonomy wasn't recreated - make sure that its lastUpdate is >= + // indexReader's lastUpdate (otherwise it means that the indexReader + // contains categories that are not yet visible to the taxonomy reader). + taxoReaderToken = getTimestampToken(taxoReader); + if (readerTaxoToken.lastUpdateTime <= taxoReaderToken.lastUpdateTime) { + pair = new SearcherTaxoPair(searcherFactory.newSearcher(reader), taxoReader); + } + } catch (InconsistentTaxonomyException e) { + // The taxonomy cannot be refreshed -- could be that someone just + // committed a 'recreate' on the taxonomy - we cannot return a new pair. + } + return pair; + } + + @Override + protected SearcherTaxoPair refreshIfNeeded(SearcherTaxoPair referenceToRefresh) throws IOException { + IndexReader newIndexReader = IndexReader.openIfChanged(referenceToRefresh.searcher.getIndexReader()); + if (newIndexReader == null) { + // search index was not updated, don't update the pair (even if the + // taxonomy index has been updated) + return null; + } + + SearcherTaxoPair newPair = null; + TaxonomyTimestamp newReaderTaxoToken = null; + TaxonomyTimestamp taxoReaderToken = null; + TaxonomyReader newTaxoReader = null; + boolean success = false; + try { + newReaderTaxoToken = getTimestampToken(newIndexReader); + taxoReaderToken = getTimestampToken(referenceToRefresh.taxoReader); + if (newReaderTaxoToken.createTime == taxoReaderToken.createTime) { + newPair = getNewPair(referenceToRefresh.taxoReader, + newIndexReader, newReaderTaxoToken, taxoReaderToken); + if (newPair != null) { + // need to incRef() because we return a new instance of + // SearcherTaxoPair, but share the taxoReader instance, and + // ReferenceManager will soon 'release' us, so need to adjust the + // reference count accordingly. + newPair.taxoReader.incRef(); + } + } else { + // the taxonomy index and search index's createTime differs. Open a new + // TaxonomyReader and try to get a matching pair with it. + newTaxoReader = new DirectoryTaxonomyReader(taxoDir); + taxoReaderToken = getTimestampToken(newTaxoReader); + if (newReaderTaxoToken.createTime == taxoReaderToken.createTime) { + newPair = getNewPair(newTaxoReader, newIndexReader, newReaderTaxoToken, taxoReaderToken); + } + } + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(newIndexReader, newTaxoReader); + } + } + + if (newPair == null) { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Search and Taxonomy indexes times do not match: searchIndex=[" + + newReaderTaxoToken + "]; taxoIndex=[" + taxoReaderToken + "]"); + } + // the taxonomy and search indexes do not match, close what was opened + IOUtils.close(newTaxoReader, newIndexReader); + } + + return newPair; + } + +} Property changes on: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/SearcherTaxoManager.java ___________________________________________________________________ Added: svn:executable ## -0,0 +1 ## +* Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TaxonomyReaderFactory.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TaxonomyReaderFactory.java (revision 0) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TaxonomyReaderFactory.java (working copy) @@ -0,0 +1,56 @@ +package org.apache.lucene.facet.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.facet.taxonomy.TaxonomyReader; +import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; +import org.apache.lucene.store.Directory; + +/** + * Factory class used by {@link SearcherTaxoManager} to create new + * TaxonomyReaders. {@link DirectoryTaxonomyReaderFactory} returns + * {@link DirectoryTaxonomyReader}s on a given {@link Directory}. + * + * @lucene.experimental + */ +public interface TaxonomyReaderFactory { + + /** + * A {@link TaxonomyReaderFactory} which returns + * {@link DirectoryTaxonomyReader}s on the given {@link Directory}. + */ + public static class DirectoryTaxonomyReaderFactory implements TaxonomyReaderFactory { + + private final Directory taxoDir; + + public DirectoryTaxonomyReaderFactory(Directory taxoDir) { + this.taxoDir = taxoDir; + } + + public TaxonomyReader newTaxonomyReader() throws IOException { + return new DirectoryTaxonomyReader(taxoDir); + } + + } + + /** Returns a new TaxonomyReader. */ + public TaxonomyReader newTaxonomyReader() throws IOException; + +} Property changes on: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TaxonomyReaderFactory.java ___________________________________________________________________ Added: svn:executable ## -0,0 +1 ## +* Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java =================================================================== --- lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (revision 1326275) +++ lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (working copy) @@ -83,14 +83,21 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { /** - * Property name of user commit data that contains the creation time of a - * taxonomy index. + * Property name of user commit data that contains the creation time (in nano + * seconds) of a taxonomy index. *

* Applications should not use this property in their commit data because it * will be overridden by this taxonomy writer. */ public static final String INDEX_CREATE_TIME = "index.create.time"; + /** + * Property name of user commit data that contains the update time (in nano + * seconds) of a taxonomy index. Set to the time that {@link #commit} is + * called. + */ + public static final String INDEX_UPDATE_TIME = "index.update.time"; + private IndexWriter indexWriter; private int nextID; private char delimiter = Consts.DEFAULT_DELIMITER; @@ -649,6 +656,9 @@ if (userData != null) { m.putAll(userData); } + // application is not allowed to override these properties, therefore set + // them last. + m.put(INDEX_UPDATE_TIME, Long.toString(System.nanoTime())); if (createTime != null) { m.put(INDEX_CREATE_TIME, createTime); }