diff -urN -x .svn -x site trunk-lusolr4/lucene/CHANGES.txt trunk-lusolr3/lucene/CHANGES.txt --- trunk-lusolr4/lucene/CHANGES.txt 2011-12-08 00:53:39.711786400 +0100 +++ trunk-lusolr3/lucene/CHANGES.txt 2011-12-08 19:31:26.432554300 +0100 @@ -207,6 +207,18 @@ * LUCENE-3533: Removed SpanFilters, they created large lists of objects and did not scale. (Robert Muir) +* LUCENE-3606: IndexReader was made read-only. It is no longer possible to + delete or undelete documents using IndexReader; you have to use IndexWriter + now. As deleting by internal Lucene docID is no longer possible, this + requires adding a unique identifier field to your index. Deleting/relying + upon Lucene docIDs is not recommended anyway, because they can change. + Consequently commit() was removed and IndexReader.open(), openIfChanged(), + and clone() no longer take readOnly booleans or IndexDeletionPolicy + instances. Furthermore, IndexReader.setNorm() was removed. If you need + customized norm values, the recommended way to do this is by modifying + SimilarityProvider to use an external byte[] or one of the new DocValues + fields (LUCENE-3108). (Uwe Schindler, Robert Muir) + Changes in Runtime Behavior * LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java trunk-lusolr3/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java --- trunk-lusolr4/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java 2011-11-25 18:56:32.543846600 +0100 +++ trunk-lusolr3/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java 2011-12-08 19:31:24.380436900 +0100 @@ -58,7 +58,7 @@ // writer.addDocument( doc ); // writer.close(); - // IndexReader reader = IndexReader.open( dir, true ); + // IndexReader reader = IndexReader.open(dir1); // new FieldTermStack( reader, 0, "f", fieldQuery ); // reader.close(); //} diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java --- trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java 2011-11-16 23:30:17.344019300 +0100 +++ trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java 2011-12-08 19:31:22.458326900 +0100 @@ -68,7 +68,7 @@ } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory, true); + final IndexReader indexReader = IndexReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -116,7 +116,7 @@ } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory, true); + final IndexReader indexReader = IndexReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -191,7 +191,7 @@ } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory, true); + final IndexReader indexReader = IndexReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -237,7 +237,7 @@ } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory, true); + final IndexReader indexReader = IndexReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -281,7 +281,7 @@ } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory, true); + final IndexReader indexReader = IndexReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java --- trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java 2011-11-16 23:30:17.356020000 +0100 +++ trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java 2011-12-08 19:31:23.089363000 +0100 @@ -1677,7 +1677,7 @@ * writer = new IndexWriter(ramDir,bigramAnalyzer , true); Document d = new * Document(); Field f = new Field(FIELD_NAME, "java abc def", true, true, * true); d.add(f); writer.addDocument(d); writer.close(); IndexReader reader = - * IndexReader.open(ramDir, true); + * IndexReader.open(ramDir); * * IndexSearcher searcher=new IndexSearcher(reader); query = * QueryParser.parse("abc", FIELD_NAME, bigramAnalyzer); @@ -1763,7 +1763,7 @@ writer.addDocument(doc, analyzer); writer.forceMerge(1); writer.close(); - reader = IndexReader.open(ramDir, true); + reader = IndexReader.open(ramDir); numHighlights = 0; } diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java --- trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java 2011-11-16 23:30:17.351019700 +0100 +++ trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java 2011-12-08 19:31:22.777345200 +0100 @@ -113,7 +113,7 @@ } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory, true); + final IndexReader indexReader = IndexReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -162,7 +162,7 @@ } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory, true); + final IndexReader indexReader = IndexReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -210,7 +210,7 @@ } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory, true); + final IndexReader indexReader = IndexReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); @@ -259,7 +259,7 @@ } finally { indexWriter.close(); } - final IndexReader indexReader = IndexReader.open(directory, true); + final IndexReader indexReader = IndexReader.open(directory); try { assertEquals(1, indexReader.numDocs()); final IndexSearcher indexSearcher = newSearcher(indexReader); diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java --- trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java 2011-11-25 14:51:56.243531500 +0100 +++ trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java 2011-12-08 19:31:23.541388900 +0100 @@ -364,7 +364,7 @@ writer.addDocument( doc ); writer.close(); if (reader != null) reader.close(); - reader = IndexReader.open( dir, true ); + reader = IndexReader.open(dir); } // make 1 doc with multi valued & not analyzed field @@ -383,7 +383,7 @@ writer.addDocument( doc ); writer.close(); if (reader != null) reader.close(); - reader = IndexReader.open( dir, true ); + reader = IndexReader.open(dir); } protected void makeIndexShortMV() throws Exception { diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java --- trunk-lusolr4/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java 2011-10-21 19:50:36.319743900 +0200 +++ trunk-lusolr3/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java 2011-12-08 19:31:23.802403800 +0100 @@ -147,7 +147,7 @@ writer.addDocument( doc ); writer.close(); if (reader != null) reader.close(); - reader = IndexReader.open( dir, true ); + reader = IndexReader.open(dir); } public void test1StrMV() throws Exception { diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java trunk-lusolr3/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java --- trunk-lusolr4/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java 2011-12-04 20:33:23.518210800 +0100 +++ trunk-lusolr3/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java 2011-12-08 19:31:19.041131500 +0100 @@ -767,11 +767,11 @@ } @Override - public int docFreq(Term term) { - Info info = getInfo(term.field()); + public int docFreq(String field, BytesRef term) { + Info info = getInfo(field); int freq = 0; - if (info != null) freq = info.getPositions(term.bytes()) != null ? 1 : 0; - if (DEBUG) System.err.println("MemoryIndexReader.docFreq: " + term + ", freq:" + freq); + if (info != null) freq = info.getPositions(term) != null ? 1 : 0; + if (DEBUG) System.err.println("MemoryIndexReader.docFreq: " + field + ":" + term + ", freq:" + freq); return freq; } @@ -1112,11 +1112,6 @@ } return norms; } - - @Override - protected void doSetNorm(int doc, String fieldName, byte value) { - throw new UnsupportedOperationException(); - } @Override public int numDocs() { @@ -1143,21 +1138,6 @@ } @Override - protected void doDelete(int docNum) { - throw new UnsupportedOperationException(); - } - - @Override - protected void doUndeleteAll() { - throw new UnsupportedOperationException(); - } - - @Override - protected void doCommit(Map commitUserData) { - if (DEBUG) System.err.println("MemoryIndexReader.doCommit"); - } - - @Override protected void doClose() { if (DEBUG) System.err.println("MemoryIndexReader.doClose"); } diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java trunk-lusolr3/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java --- trunk-lusolr4/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java 2011-12-04 20:33:23.635217500 +0100 +++ trunk-lusolr3/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,166 +0,0 @@ -package org.apache.lucene.index; - -/** - * Copyright 2006 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.io.File; -import java.util.Date; -import java.util.List; -import java.util.ArrayList; - -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarities.SimilarityProvider; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.ReaderUtil; - -/** - * Given a directory and a list of fields, updates the fieldNorms in place for every document. - * - * If Similarity class is specified, uses its computeNorm method to set norms. - * If -n command line argument is used, removed field norms, as if - * {@link org.apache.lucene.document.FieldType#setOmitNorms(boolean)} was used. - * - *

- * NOTE: This will overwrite any length normalization or field/document boosts. - *

- * - */ -public class FieldNormModifier { - - /** - * Command Line Execution method. - * - * 
-   * Usage: FieldNormModifier /path/index <package.SimilarityClassName | -n> field1 field2 ...
-   *
- */ - public static void main(String[] args) throws IOException { - if (args.length < 3) { - System.err.println("Usage: FieldNormModifier [field2] ..."); - System.exit(1); - } - - SimilarityProvider s = null; - - if (args[1].equals("-d")) - args[1] = DefaultSimilarity.class.getName(); - - try { - s = Class.forName(args[1]).asSubclass(SimilarityProvider.class).newInstance(); - } catch (Exception e) { - System.err.println("Couldn't instantiate similarity with empty constructor: " + args[1]); - e.printStackTrace(System.err); - System.exit(1); - } - - Directory d = FSDirectory.open(new File(args[0])); - FieldNormModifier fnm = new FieldNormModifier(d, s); - - for (int i = 2; i < args.length; i++) { - System.out.print("Updating field: " + args[i] + " " + (new Date()).toString() + " ... "); - fnm.reSetNorms(args[i]); - System.out.println(new Date().toString()); - } - - d.close(); - } - - - private Directory dir; - private SimilarityProvider sim; - - /** - * Constructor for code that wishes to use this class programmatically - * If Similarity is null, kill the field norms. - * - * @param d the Directory to modify - * @param s the Similarity to use (can be null) - */ - public FieldNormModifier(Directory d, SimilarityProvider s) { - dir = d; - sim = s; - } - - /** - * Resets the norms for the specified field. - * - *

- * Opens a new IndexReader on the Directory given to this instance, - * modifies the norms (either using the Similarity given to this instance, or by using fake norms, - * and closes the IndexReader. - *

- * - * @param field the field whose norms should be reset - */ - public void reSetNorms(String field) throws IOException { - Similarity fieldSim = sim.get(field); - IndexReader reader = null; - try { - reader = IndexReader.open(dir, false); - - final List subReaders = new ArrayList(); - ReaderUtil.gatherSubReaders(subReaders, reader); - - final FieldInvertState invertState = new FieldInvertState(); - for(IndexReader subReader : subReaders) { - final Bits liveDocs = subReader.getLiveDocs(); - - int[] termCounts = new int[subReader.maxDoc()]; - Fields fields = subReader.fields(); - if (fields != null) { - Terms terms = fields.terms(field); - if (terms != null) { - TermsEnum termsEnum = terms.iterator(null); - DocsEnum docs = null; - DocsEnum docsAndFreqs = null; - while(termsEnum.next() != null) { - docsAndFreqs = termsEnum.docs(liveDocs, docsAndFreqs, true); - final DocsEnum docs2; - if (docsAndFreqs != null) { - docs2 = docsAndFreqs; - } else { - docs2 = docs = termsEnum.docs(liveDocs, docs, false); - } - while(true) { - int docID = docs2.nextDoc(); - if (docID != docs.NO_MORE_DOCS) { - termCounts[docID] += docsAndFreqs == null ? 1 : docsAndFreqs.freq(); - } else { - break; - } - } - } - } - } - - invertState.setBoost(1.0f); - for (int d = 0; d < termCounts.length; d++) { - if (liveDocs == null || liveDocs.get(d)) { - invertState.setLength(termCounts[d]); - subReader.setNorm(d, field, fieldSim.computeNorm(invertState)); - } - } - } - - } finally { - if (null != reader) reader.close(); - } - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java trunk-lusolr3/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java --- trunk-lusolr4/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java 2011-10-19 14:21:26.579673900 +0200 +++ trunk-lusolr3/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java 2011-12-08 19:31:21.083248300 +0100 @@ -49,7 +49,7 @@ /** * Split source index into multiple parts. - * @param input source index, can be read-only, can have deletions, can have + * @param in source index, can have deletions, can have * multiple segments (or multiple readers). * @param outputs list of directories where the output parts will be stored. * @param seq if true, then the source index will be split into equal @@ -57,18 +57,18 @@ * assigned in a deterministic round-robin fashion to one of the output splits. * @throws IOException */ - public void split(Version version, IndexReader input, Directory[] outputs, boolean seq) throws IOException { + public void split(Version version, IndexReader in, Directory[] outputs, boolean seq) throws IOException { if (outputs == null || outputs.length < 2) { throw new IOException("Invalid number of outputs."); } - if (input == null || input.numDocs() < 2) { + if (in == null || in.numDocs() < 2) { throw new IOException("Not enough documents for splitting"); } int numParts = outputs.length; // wrap a potentially read-only input // this way we don't have to preserve original deletions because neither // deleteDocument(int) or undeleteAll() is applied to the wrapped input index. - input = new FakeDeleteIndexReader(input); + FakeDeleteIndexReader input = new FakeDeleteIndexReader(in); int maxDoc = input.maxDoc(); int partLen = maxDoc / numParts; for (int i = 0; i < numParts; i++) { @@ -143,7 +143,7 @@ System.err.println("Invalid input index - skipping: " + file); continue; } - indexes.add(IndexReader.open(dir, true)); + indexes.add(IndexReader.open(dir)); } } if (outDir == null) { @@ -183,7 +183,7 @@ public FakeDeleteIndexReader(IndexReader in) { super(new SlowMultiReaderWrapper(in)); - doUndeleteAll(); // initialize main bitset + undeleteAll(); // initialize main bitset } @Override @@ -191,12 +191,7 @@ return liveDocs.cardinality(); } - /** - * Just removes our overlaid deletions - does not undelete the original - * deletions. - */ - @Override - protected void doUndeleteAll() { + void undeleteAll() { final int maxDoc = in.maxDoc(); liveDocs = new FixedBitSet(in.maxDoc()); if (in.hasDeletions()) { @@ -212,8 +207,7 @@ } } - @Override - protected void doDelete(int n) { + void deleteDocument(int n) { liveDocs.clear(n); } diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java trunk-lusolr3/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java --- trunk-lusolr4/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java 2011-12-04 20:33:23.665219200 +0100 +++ trunk-lusolr3/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java 2011-12-08 19:31:21.289260100 +0100 @@ -78,7 +78,7 @@ } } - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); TermStats[] terms = getHighFreqTerms(reader, numTerms, field); if (!IncludeTermFreqs) { //default HighFreqTerms behavior diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java --- trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java 2011-11-16 23:30:15.232898600 +0100 +++ trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,259 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.search.similarities.DefaultSimilarityProvider; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarities.SimilarityProvider; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; - -/** - * Tests changing of field norms with a custom similarity and with fake norms. - */ -public class TestFieldNormModifier extends LuceneTestCase { - public static int NUM_DOCS = 5; - - public Directory store; - - /** inverts the normal notion of lengthNorm */ - public static SimilarityProvider s = new DefaultSimilarityProvider() { - @Override - public Similarity get(String field) { - return new DefaultSimilarity() { - @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength())); - } - }; - } - }; - - @Override - public void setUp() throws Exception { - super.setUp(); - store = newDirectory(); - IndexWriter writer = new IndexWriter(store, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); - - for (int i = 0; i < NUM_DOCS; i++) { - Document d = new Document(); - - d.add(newField("field", "word", TextField.TYPE_STORED)); - - d.add(newField("nonorm", "word", StringField.TYPE_STORED)); - d.add(newField("untokfield", "20061212 20071212", TextField.TYPE_STORED)); - - for (int j = 1; j <= i; j++) { - d.add(newField("field", "crap", TextField.TYPE_STORED)); - d.add(newField("nonorm", "more words", StringField.TYPE_STORED)); - } - writer.addDocument(d); - } - writer.close(); - } - - @Override - public void tearDown() throws Exception { - store.close(); - super.tearDown(); - } - - public void testMissingField() throws Exception { - FieldNormModifier fnm = new FieldNormModifier(store, s); - try { - fnm.reSetNorms("nobodyherebutuschickens"); - } catch (IllegalStateException e) { - // expected - } - } - - public void testFieldWithNoNorm() throws Exception { - - IndexReader r = IndexReader.open(store, false); - byte[] norms = MultiNorms.norms(r, "nonorm"); - - // sanity check, norms should all be 1 - assertTrue("Whoops we have norms?", !r.hasNorms("nonorm")); - assertNull(norms); - - r.close(); - - FieldNormModifier fnm = new FieldNormModifier(store, s); - try { - fnm.reSetNorms("nonorm"); - } catch (IllegalStateException e) { - // expected - } - - // nothing should have changed - r = IndexReader.open(store, false); - - norms = MultiNorms.norms(r, "nonorm"); - assertTrue("Whoops we have norms?", !r.hasNorms("nonorm")); - assertNull(norms); - - r.close(); - } - - - public void testGoodCases() throws Exception { - - IndexReader reader = IndexReader.open(store); - IndexSearcher searcher = new IndexSearcher(reader); - final float[] scores = new float[NUM_DOCS]; - float lastScore = 0.0f; - - // default similarity should put docs with shorter length first - searcher.search(new TermQuery(new Term("field", "word")), new Collector() { - private int docBase = 0; - private Scorer scorer; - - @Override - public final void collect(int doc) throws IOException { - scores[doc + docBase] = scorer.score(); - } - @Override - public void setNextReader(AtomicReaderContext context) { - docBase = context.docBase; - } - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - searcher.close(); - reader.close(); - - lastScore = Float.MAX_VALUE; - for (int i = 0; i < NUM_DOCS; i++) { - String msg = "i=" + i + ", " + scores[i] + " <= " + lastScore; - assertTrue(msg, scores[i] <= lastScore); - //System.out.println(msg); - lastScore = scores[i]; - } - - FieldNormModifier fnm = new FieldNormModifier(store, s); - fnm.reSetNorms("field"); - - // new norm (with default similarity) should put longer docs first - reader = IndexReader.open(store); - searcher = new IndexSearcher(reader); - searcher.search(new TermQuery(new Term("field", "word")), new Collector() { - private int docBase = 0; - private Scorer scorer; - @Override - public final void collect(int doc) throws IOException { - scores[doc + docBase] = scorer.score(); - } - @Override - public void setNextReader(AtomicReaderContext context) { - docBase = context.docBase; - } - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - searcher.close(); - reader.close(); - - lastScore = 0.0f; - for (int i = 0; i < NUM_DOCS; i++) { - String msg = "i=" + i + ", " + scores[i] + " >= " + lastScore; - assertTrue(msg, scores[i] >= lastScore); - //System.out.println(msg); - lastScore = scores[i]; - } - } - - public void testNormKiller() throws IOException { - - IndexReader r = IndexReader.open(store, false); - byte[] oldNorms = MultiNorms.norms(r, "untokfield"); - r.close(); - - FieldNormModifier fnm = new FieldNormModifier(store, s); - fnm.reSetNorms("untokfield"); - - r = IndexReader.open(store, false); - byte[] newNorms = MultiNorms.norms(r, "untokfield"); - r.close(); - assertFalse(Arrays.equals(oldNorms, newNorms)); - - - // verify that we still get documents in the same order as originally - IndexReader reader = IndexReader.open(store); - IndexSearcher searcher = new IndexSearcher(reader); - final float[] scores = new float[NUM_DOCS]; - float lastScore = 0.0f; - - // default similarity should return the same score for all documents for this query - searcher.search(new TermQuery(new Term("untokfield", "20061212")), new Collector() { - private int docBase = 0; - private Scorer scorer; - @Override - public final void collect(int doc) throws IOException { - scores[doc + docBase] = scorer.score(); - } - @Override - public void setNextReader(AtomicReaderContext context) { - docBase = context.docBase; - } - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - searcher.close(); - reader.close(); - - lastScore = scores[0]; - for (int i = 0; i < NUM_DOCS; i++) { - String msg = "i=" + i + ", " + scores[i] + " == " + lastScore; - assertTrue(msg, scores[i] == lastScore); - //System.out.println(msg); - lastScore = scores[i]; - } - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java --- trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java 2011-11-12 00:38:42.012898800 +0100 +++ trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java 2011-12-08 19:31:19.961184100 +0100 @@ -69,7 +69,7 @@ String splitSegName = is.infos.info(1).name; is.split(destDir, new String[] {splitSegName}); Directory fsDirDest = newFSDirectory(destDir); - IndexReader r = IndexReader.open(fsDirDest, true); + IndexReader r = IndexReader.open(fsDirDest); assertEquals(50, r.maxDoc()); r.close(); fsDirDest.close(); @@ -81,76 +81,17 @@ IndexSplitter.main(new String[] {dir.getAbsolutePath(), destDir2.getAbsolutePath(), splitSegName}); assertEquals(4, destDir2.listFiles().length); Directory fsDirDest2 = newFSDirectory(destDir2); - r = IndexReader.open(fsDirDest2, true); + r = IndexReader.open(fsDirDest2); assertEquals(50, r.maxDoc()); r.close(); fsDirDest2.close(); // now remove the copied segment from src IndexSplitter.main(new String[] {dir.getAbsolutePath(), "-d", splitSegName}); - r = IndexReader.open(fsDir, true); + r = IndexReader.open(fsDir); assertEquals(2, r.getSequentialSubReaders().length); r.close(); fsDir.close(); } - public void testDeleteThenFullMerge() throws Exception { - // Create directories where the indexes will reside - File indexPath = new File(TEMP_DIR, "testfilesplitter"); - _TestUtil.rmDir(indexPath); - indexPath.mkdirs(); - File indexSplitPath = new File(TEMP_DIR, "testfilesplitterdest"); - _TestUtil.rmDir(indexSplitPath); - indexSplitPath.mkdirs(); - - // Create the original index - LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); - mergePolicy.setNoCFSRatio(1); - IndexWriterConfig iwConfig - = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) - .setOpenMode(OpenMode.CREATE) - .setMergePolicy(mergePolicy); - Directory fsDir = newFSDirectory(indexPath); - IndexWriter indexWriter = new IndexWriter(fsDir, iwConfig); - Document doc = new Document(); - doc.add(new Field("content", "doc 1", StringField.TYPE_STORED)); - indexWriter.addDocument(doc); - doc = new Document(); - doc.add(new Field("content", "doc 2", StringField.TYPE_STORED)); - indexWriter.addDocument(doc); - indexWriter.close(); - fsDir.close(); - - // Create the split index - IndexSplitter indexSplitter = new IndexSplitter(indexPath); - String splitSegName = indexSplitter.infos.info(0).name; - indexSplitter.split(indexSplitPath, new String[] {splitSegName}); - - // Delete the first document in the split index - Directory fsDirDest = newFSDirectory(indexSplitPath); - IndexReader indexReader = IndexReader.open(fsDirDest, false); - indexReader.deleteDocument(0); - assertEquals(1, indexReader.numDocs()); - indexReader.close(); - fsDirDest.close(); - - // Fully merge the split index - mergePolicy = new LogByteSizeMergePolicy(); - mergePolicy.setNoCFSRatio(1); - iwConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) - .setOpenMode(OpenMode.APPEND) - .setMergePolicy(mergePolicy); - fsDirDest = newFSDirectory(indexSplitPath); - indexWriter = new IndexWriter(fsDirDest, iwConfig); - indexWriter.forceMerge(1); - indexWriter.close(); - fsDirDest.close(); - - // Read the number of docs in the index - fsDirDest = newFSDirectory(indexSplitPath); - indexReader = IndexReader.open(fsDirDest); - assertEquals(1, indexReader.numDocs()); - indexReader.close(); - fsDirDest.close(); - } } diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java --- trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java 2011-11-16 23:30:15.158894300 +0100 +++ trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java 2011-12-08 19:31:19.830176600 +0100 @@ -41,10 +41,10 @@ doc.add(newField("f", i + " " + i, TextField.TYPE_STORED)); w.addDocument(doc); } + w.commit(); + w.deleteDocuments(new Term("id", "" + (NUM_DOCS-1))); w.close(); - input = IndexReader.open(dir, false); - // delete the last doc - input.deleteDocument(input.maxDoc() - 1); + input = IndexReader.open(dir); } @Override @@ -66,7 +66,7 @@ }; splitter.split(TEST_VERSION_CURRENT, input, dirs, false); IndexReader ir; - ir = IndexReader.open(dirs[0], true); + ir = IndexReader.open(dirs[0]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); // rounding error Document doc = ir.document(0); assertEquals("0", doc.get("id")); @@ -74,7 +74,7 @@ assertEquals(TermsEnum.SeekStatus.NOT_FOUND, te.seekCeil(new BytesRef("1"))); assertNotSame("1", te.term().utf8ToString()); ir.close(); - ir = IndexReader.open(dirs[1], true); + ir = IndexReader.open(dirs[1]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); doc = ir.document(0); assertEquals("1", doc.get("id")); @@ -83,7 +83,7 @@ assertNotSame("0", te.term().utf8ToString()); ir.close(); - ir = IndexReader.open(dirs[2], true); + ir = IndexReader.open(dirs[2]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); doc = ir.document(0); assertEquals("2", doc.get("id")); @@ -111,19 +111,19 @@ }; splitter.split(TEST_VERSION_CURRENT, input, dirs, true); IndexReader ir; - ir = IndexReader.open(dirs[0], true); + ir = IndexReader.open(dirs[0]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); Document doc = ir.document(0); assertEquals("0", doc.get("id")); int start = ir.numDocs(); ir.close(); - ir = IndexReader.open(dirs[1], true); + ir = IndexReader.open(dirs[1]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); doc = ir.document(0); assertEquals(start + "", doc.get("id")); start += ir.numDocs(); ir.close(); - ir = IndexReader.open(dirs[2], true); + ir = IndexReader.open(dirs[2]); assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); doc = ir.document(0); assertEquals(start + "", doc.get("id")); diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java --- trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java 2011-11-10 16:27:06.317623100 +0100 +++ trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java 2011-12-08 19:31:20.512215600 +0100 @@ -43,7 +43,7 @@ TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)) .setMaxBufferedDocs(2)); indexDocs(writer); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); _TestUtil.checkIndex(dir); } diff -urN -x .svn -x site trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java --- trunk-lusolr4/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java 2011-11-16 23:30:15.399908100 +0100 +++ trunk-lusolr3/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,222 +0,0 @@ -package org.apache.lucene.misc; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.FieldNormModifier; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.MultiNorms; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.search.similarities.DefaultSimilarityProvider; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarities.SimilarityProvider; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; - -/** - * Tests changing the norms after changing the simularity - */ -public class TestLengthNormModifier extends LuceneTestCase { - public static int NUM_DOCS = 5; - - public Directory store; - - /** inverts the normal notion of lengthNorm */ - public static SimilarityProvider s = new DefaultSimilarityProvider() { - @Override - public Similarity get(String field) { - return new DefaultSimilarity() { - @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength())); - } - }; - } - }; - - @Override - public void setUp() throws Exception { - super.setUp(); - store = newDirectory(); - IndexWriter writer = new IndexWriter(store, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); - - for (int i = 0; i < NUM_DOCS; i++) { - Document d = new Document(); - d.add(newField("field", "word", TextField.TYPE_STORED)); - d.add(newField("nonorm", "word", StringField.TYPE_STORED)); - - for (int j = 1; j <= i; j++) { - d.add(newField("field", "crap", TextField.TYPE_STORED)); - d.add(newField("nonorm", "more words", StringField.TYPE_STORED)); - } - writer.addDocument(d); - } - writer.close(); - } - - @Override - public void tearDown() throws Exception { - store.close(); - super.tearDown(); - } - - public void testMissingField() throws Exception { - FieldNormModifier fnm = new FieldNormModifier(store, s); - try { - fnm.reSetNorms("nobodyherebutuschickens"); - } catch (IllegalStateException e) { - // expected - } - } - - public void testFieldWithNoNorm() throws Exception { - - IndexReader r = IndexReader.open(store, false); - byte[] norms = MultiNorms.norms(r, "nonorm"); - - // sanity check, norms should all be 1 - assertTrue("Whoops we have norms?", !r.hasNorms("nonorm")); - assertNull(norms); - - r.close(); - - FieldNormModifier fnm = new FieldNormModifier(store, s); - try { - fnm.reSetNorms("nonorm"); - } catch (IllegalStateException e) { - // expected - } - - // nothing should have changed - r = IndexReader.open(store, false); - - norms = MultiNorms.norms(r, "nonorm"); - assertTrue("Whoops we have norms?", !r.hasNorms("nonorm")); - assertNull(norms); - - r.close(); - - } - - - public void testGoodCases() throws Exception { - - IndexSearcher searcher; - final float[] scores = new float[NUM_DOCS]; - float lastScore = 0.0f; - - // default similarity should put docs with shorter length first - IndexReader reader = IndexReader.open(store, false); - searcher = new IndexSearcher(reader); - searcher.search(new TermQuery(new Term("field", "word")), new Collector() { - private int docBase = 0; - private Scorer scorer; - @Override - public final void collect(int doc) throws IOException { - scores[doc + docBase] = scorer.score(); - } - @Override - public void setNextReader(AtomicReaderContext context) { - docBase = context.docBase; - } - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - searcher.close(); - reader.close(); - - lastScore = Float.MAX_VALUE; - for (int i = 0; i < NUM_DOCS; i++) { - String msg = "i=" + i + ", "+scores[i]+" <= "+lastScore; - assertTrue(msg, scores[i] <= lastScore); - //System.out.println(msg); - lastScore = scores[i]; - } - - // override the norms to be inverted - SimilarityProvider s = new DefaultSimilarityProvider() { - @Override - public Similarity get(String field) { - return new DefaultSimilarity() { - @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength())); - } - }; - } - }; - - FieldNormModifier fnm = new FieldNormModifier(store, s); - fnm.reSetNorms("field"); - - // new norm (with default similarity) should put longer docs first - reader = IndexReader.open(store, false); - searcher = new IndexSearcher(reader); - searcher.search(new TermQuery(new Term("field", "word")), new Collector() { - private int docBase = 0; - private Scorer scorer; - @Override - public final void collect(int doc) throws IOException { - scores[doc + docBase] = scorer.score(); - } - @Override - public void setNextReader(AtomicReaderContext context) { - docBase = context.docBase; - } - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - searcher.close(); - reader.close(); - - lastScore = 0.0f; - for (int i = 0; i < NUM_DOCS; i++) { - String msg = "i=" + i + ", "+scores[i]+" >= "+lastScore; - assertTrue(msg, scores[i] >= lastScore); - //System.out.println(msg); - lastScore = scores[i]; - } - - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/CheckIndex.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/CheckIndex.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/CheckIndex.java 2011-12-04 20:33:24.891289400 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/CheckIndex.java 2011-12-08 19:31:51.064963200 +0100 @@ -524,7 +524,7 @@ } if (infoStream != null) infoStream.print(" test: open reader........."); - reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.DEFAULT); + reader = SegmentReader.get(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.DEFAULT); segInfoStat.openReaderPassed = true; diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/DirectoryReader.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/DirectoryReader.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/DirectoryReader.java 2011-12-04 17:19:06.887489700 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/DirectoryReader.java 2011-12-08 19:31:48.034789800 +0100 @@ -44,18 +44,12 @@ */ class DirectoryReader extends IndexReader implements Cloneable { protected Directory directory; - protected boolean readOnly; IndexWriter writer; - private IndexDeletionPolicy deletionPolicy; - private Lock writeLock; private final SegmentInfos segmentInfos; - private boolean stale; private final int termInfosIndexDivisor; - private boolean rollbackHasChanges; - private SegmentReader[] subReaders; private ReaderContext topLevelReaderContext; private int[] starts; // 1st docno for each segment @@ -63,41 +57,24 @@ private int numDocs = -1; private boolean hasDeletions = false; - // Max version in index as of when we opened; this can be - // > our current segmentInfos version in case we were - // opened on a past IndexCommit: - private long maxIndexVersion; - private final boolean applyAllDeletes; -// static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, -// final int termInfosIndexDivisor) throws CorruptIndexException, IOException { -// return open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); -// } - - static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, + public static IndexReader open(final Directory directory, final IndexCommit commit, final int termInfosIndexDivisor) throws CorruptIndexException, IOException { return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { SegmentInfos infos = new SegmentInfos(); infos.read(directory, segmentFileName); - return new DirectoryReader(directory, infos, deletionPolicy, readOnly, termInfosIndexDivisor); + return new DirectoryReader(directory, infos, termInfosIndexDivisor); } }.run(commit); } - - /** Construct reading the named set of readers. */ -// DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { -// this(directory, sis, deletionPolicy, readOnly, termInfosIndexDivisor, null); -// } /** Construct reading the named set of readers. */ - DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { + DirectoryReader(Directory directory, SegmentInfos sis, int termInfosIndexDivisor) throws IOException { this.directory = directory; - this.readOnly = readOnly; this.segmentInfos = sis; - this.deletionPolicy = deletionPolicy; this.termInfosIndexDivisor = termInfosIndexDivisor; readerFinishedListeners = new MapBackedSet(new ConcurrentHashMap()); applyAllDeletes = false; @@ -111,7 +88,7 @@ for (int i = sis.size()-1; i >= 0; i--) { boolean success = false; try { - readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor, IOContext.READ); + readers[i] = SegmentReader.get(sis.info(i), termInfosIndexDivisor, IOContext.READ); readers[i].readerFinishedListeners = readerFinishedListeners; success = true; } finally { @@ -134,7 +111,6 @@ // Used by near real-time search DirectoryReader(IndexWriter writer, SegmentInfos infos, boolean applyAllDeletes) throws IOException { this.directory = writer.getDirectory(); - this.readOnly = true; this.applyAllDeletes = applyAllDeletes; // saved for reopen this.termInfosIndexDivisor = writer.getConfig().getReaderTermsIndexDivisor(); @@ -186,9 +162,8 @@ /** This constructor is only used for {@link #doOpenIfChanged()} */ DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, - boolean readOnly, boolean doClone, int termInfosIndexDivisor, Collection readerFinishedListeners) throws IOException { + boolean doClone, int termInfosIndexDivisor, Collection readerFinishedListeners) throws IOException { this.directory = directory; - this.readOnly = readOnly; this.segmentInfos = infos; this.termInfosIndexDivisor = termInfosIndexDivisor; this.readerFinishedListeners = readerFinishedListeners; @@ -231,12 +206,12 @@ assert !doClone; // this is a new reader; in case we hit an exception we can close it safely - newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor, IOContext.READ); + newReader = SegmentReader.get(infos.info(i), termInfosIndexDivisor, IOContext.READ); newReader.readerFinishedListeners = readerFinishedListeners; readerShared[i] = false; newReaders[i] = newReader; } else { - newReader = newReaders[i].reopenSegment(infos.info(i), doClone, readOnly); + newReader = newReaders[i].reopenSegment(infos.info(i), doClone); if (newReader == null) { // this reader will be shared between the old and the new one, // so we must incRef it @@ -281,9 +256,6 @@ @Override public String toString() { final StringBuilder buffer = new StringBuilder(); - if (hasChanges) { - buffer.append("*"); - } buffer.append(getClass().getSimpleName()); buffer.append('('); final String segmentsFile = segmentInfos.getCurrentSegmentFileName(); @@ -323,10 +295,6 @@ } } starts[subReaders.length] = maxDoc; - - if (!readOnly) { - maxIndexVersion = SegmentInfos.readCurrentVersion(directory); - } } @Override @@ -337,69 +305,46 @@ @Override public final synchronized Object clone() { try { - return clone(readOnly); // Preserve current readOnly + DirectoryReader newReader = doOpenIfChanged((SegmentInfos) segmentInfos.clone(), true); + newReader.writer = writer; + newReader.hasDeletions = hasDeletions; + assert newReader.readerFinishedListeners != null; + return newReader; } catch (Exception ex) { throw new RuntimeException(ex); } } @Override - public final synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException { - // doOpenIfChanged calls ensureOpen - DirectoryReader newReader = doOpenIfChanged((SegmentInfos) segmentInfos.clone(), true, openReadOnly); - - if (this != newReader) { - newReader.deletionPolicy = deletionPolicy; - } - newReader.writer = writer; - // If we're cloning a non-readOnly reader, move the - // writeLock (if there is one) to the new reader: - if (!openReadOnly && writeLock != null) { - // In near real-time search, reader is always readonly - assert writer == null; - newReader.writeLock = writeLock; - newReader.hasChanges = hasChanges; - newReader.hasDeletions = hasDeletions; - writeLock = null; - hasChanges = false; - } - assert newReader.readerFinishedListeners != null; - - return newReader; - } - - @Override protected final IndexReader doOpenIfChanged() throws CorruptIndexException, IOException { - // Preserve current readOnly - return doOpenIfChanged(readOnly, null); - } - - @Override - protected final IndexReader doOpenIfChanged(boolean openReadOnly) throws CorruptIndexException, IOException { - return doOpenIfChanged(openReadOnly, null); + return doOpenIfChanged(null); } @Override protected final IndexReader doOpenIfChanged(final IndexCommit commit) throws CorruptIndexException, IOException { - return doOpenIfChanged(true, commit); + ensureOpen(); + + // If we were obtained by writer.getReader(), re-ask the + // writer to get a new reader. + if (writer != null) { + return doOpenFromWriter(commit); + } else { + return doOpenNoWriter(commit); + } } @Override protected final IndexReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException { + ensureOpen(); if (writer == this.writer && applyAllDeletes == this.applyAllDeletes) { - return doOpenIfChanged(); - } else { + return doOpenFromWriter(null); + } else { + // fail by calling supers impl throwing UOE return super.doOpenIfChanged(writer, applyAllDeletes); } } - private final IndexReader doOpenFromWriter(boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException { - assert readOnly; - - if (!openReadOnly) { - throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)"); - } - + private final IndexReader doOpenFromWriter(IndexCommit commit) throws CorruptIndexException, IOException { if (commit != null) { throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit"); } @@ -420,56 +365,18 @@ return reader; } - private IndexReader doOpenIfChanged(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException { - ensureOpen(); - - assert commit == null || openReadOnly; - - // If we were obtained by writer.getReader(), re-ask the - // writer to get a new reader. - if (writer != null) { - return doOpenFromWriter(openReadOnly, commit); - } else { - return doOpenNoWriter(openReadOnly, commit); - } - } - - private synchronized IndexReader doOpenNoWriter(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException { + private synchronized IndexReader doOpenNoWriter(IndexCommit commit) throws CorruptIndexException, IOException { if (commit == null) { - if (hasChanges) { - // We have changes, which means we are not readOnly: - assert readOnly == false; - // and we hold the write lock: - assert writeLock != null; - // so no other writer holds the write lock, which - // means no changes could have been done to the index: - assert isCurrent(); - - if (openReadOnly) { - return clone(openReadOnly); - } else { - return null; - } - } else if (isCurrent()) { - if (openReadOnly != readOnly) { - // Just fallback to clone - return clone(openReadOnly); - } else { - return null; - } + if (isCurrent()) { + return null; } } else { if (directory != commit.getDirectory()) { throw new IOException("the specified commit does not match the specified Directory"); } if (segmentInfos != null && commit.getSegmentsFileName().equals(segmentInfos.getCurrentSegmentFileName())) { - if (readOnly != openReadOnly) { - // Just fallback to clone - return clone(openReadOnly); - } else { - return null; - } + return null; } } @@ -478,13 +385,13 @@ protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { final SegmentInfos infos = new SegmentInfos(); infos.read(directory, segmentFileName); - return doOpenIfChanged(infos, false, openReadOnly); + return doOpenIfChanged(infos, false); } }.run(commit); } - private synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { - return new DirectoryReader(directory, infos, subReaders, openReadOnly, doClone, termInfosIndexDivisor, readerFinishedListeners); + private synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, boolean doClone) throws CorruptIndexException, IOException { + return new DirectoryReader(directory, infos, subReaders, doClone, termInfosIndexDivisor, readerFinishedListeners); } /** Version number when this IndexReader was opened. */ @@ -535,23 +442,6 @@ return hasDeletions; } - @Override - protected void doDelete(int n) throws CorruptIndexException, IOException { - numDocs = -1; // invalidate cache - int i = readerIndex(n); // find segment num - subReaders[i].deleteDocument(n - starts[i]); // dispatch to segment reader - hasDeletions = true; - } - - @Override - protected void doUndeleteAll() throws CorruptIndexException, IOException { - for (int i = 0; i < subReaders.length; i++) - subReaders[i].undeleteAll(); - - hasDeletions = false; - numDocs = -1; // invalidate cache - } - private int readerIndex(int n) { // find reader for doc n: return readerIndex(n, this.starts, this.subReaders.length); } @@ -593,22 +483,6 @@ } @Override - protected void doSetNorm(int n, String field, byte value) - throws CorruptIndexException, IOException { - int i = readerIndex(n); // find segment num - subReaders[i].setNorm(n-starts[i], field, value); // dispatch - } - - @Override - public int docFreq(Term t) throws IOException { - ensureOpen(); - int total = 0; // sum freqs in segments - for (int i = 0; i < subReaders.length; i++) - total += subReaders[i].docFreq(t); - return total; - } - - @Override public int docFreq(String field, BytesRef term) throws IOException { ensureOpen(); int total = 0; // sum freqs in segments @@ -623,150 +497,6 @@ throw new UnsupportedOperationException("please use MultiFields.getFields, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Fields"); } - /** - * Tries to acquire the WriteLock on this directory. this method is only valid if this IndexReader is directory - * owner. - * - * @throws StaleReaderException if the index has changed since this reader was opened - * @throws CorruptIndexException if the index is corrupt - * @throws org.apache.lucene.store.LockObtainFailedException - * if another writer has this index open (write.lock could not be - * obtained) - * @throws IOException if there is a low-level IO error - */ - @Override - protected void acquireWriteLock() throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException { - - if (readOnly) { - // NOTE: we should not reach this code w/ the core - // IndexReader classes; however, an external subclass - // of IndexReader could reach this. - throw new UnsupportedOperationException("This IndexReader cannot make any changes to the index (it was opened with readOnly = true)"); - } - - if (segmentInfos != null) { - ensureOpen(); - if (stale) - throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations"); - - if (writeLock == null) { - Lock writeLock = directory.makeLock(IndexWriter.WRITE_LOCK_NAME); - if (!writeLock.obtain(IndexWriterConfig.WRITE_LOCK_TIMEOUT)) // obtain write lock - throw new LockObtainFailedException("Index locked for write: " + writeLock); - this.writeLock = writeLock; - - // we have to check whether index has changed since this reader was opened. - // if so, this reader is no longer valid for deletion - if (SegmentInfos.readCurrentVersion(directory) > maxIndexVersion) { - stale = true; - this.writeLock.release(); - this.writeLock = null; - throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations"); - } - } - } - } - - /** - * Commit changes resulting from delete, undeleteAll, or setNorm operations - *

- * If an exception is hit, then either no changes or all changes will have been committed to the index (transactional - * semantics). - * - * @throws IOException if there is a low-level IO error - */ - @Override - protected void doCommit(Map commitUserData) throws IOException { - // poll subreaders for changes - for (int i = 0; !hasChanges && i < subReaders.length; i++) { - hasChanges |= subReaders[i].hasChanges; - } - - if (hasChanges) { - segmentInfos.setUserData(commitUserData); - // Default deleter (for backwards compatibility) is - // KeepOnlyLastCommitDeleter: - // TODO: Decide what to do with InfoStream here? Use default or keep NO_OUTPUT? - IndexFileDeleter deleter = new IndexFileDeleter(directory, - deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, InfoStream.NO_OUTPUT, null); - segmentInfos.updateGeneration(deleter.getLastSegmentInfos()); - segmentInfos.changed(); - - // Checkpoint the state we are about to change, in - // case we have to roll back: - startCommit(); - - final List rollbackSegments = segmentInfos.createBackupSegmentInfos(false); - - boolean success = false; - try { - for (int i = 0; i < subReaders.length; i++) - subReaders[i].commit(); - - // Remove segments that contain only 100% deleted - // docs: - segmentInfos.pruneDeletedSegments(); - - // Sync all files we just wrote - directory.sync(segmentInfos.files(directory, false)); - segmentInfos.commit(directory, segmentInfos.codecFormat()); - success = true; - } finally { - - if (!success) { - - // Rollback changes that were made to - // SegmentInfos but failed to get [fully] - // committed. This way this reader instance - // remains consistent (matched to what's - // actually in the index): - rollbackCommit(); - - // Recompute deletable files & remove them (so - // partially written .del files, etc, are - // removed): - deleter.refresh(); - - // Restore all SegmentInfos (in case we pruned some) - segmentInfos.rollbackSegmentInfos(rollbackSegments); - } - } - - // Have the deleter remove any now unreferenced - // files due to this commit: - deleter.checkpoint(segmentInfos, true); - deleter.close(); - - maxIndexVersion = segmentInfos.getVersion(); - - if (writeLock != null) { - writeLock.release(); // release write lock - writeLock = null; - } - } - hasChanges = false; - } - - void startCommit() { - rollbackHasChanges = hasChanges; - for (int i = 0; i < subReaders.length; i++) { - subReaders[i].startCommit(); - } - } - - void rollbackCommit() { - hasChanges = rollbackHasChanges; - for (int i = 0; i < subReaders.length; i++) { - subReaders[i].rollbackCommit(); - } - } - - @Override - public long getUniqueTermCount() throws IOException { - return -1; - } - @Override public Map getCommitUserData() { ensureOpen(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java 2011-12-03 13:28:28.870214100 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java 2011-12-08 19:31:50.627938200 +0100 @@ -52,7 +52,7 @@ * are processing the document). * * Other consumers, eg {@link FreqProxTermsWriter} and - * {@link NormsWriter}, buffer bytes in RAM and flush only + * {@link NormsConsumer}, buffer bytes in RAM and flush only * when a new segment is produced. * Once we have used our allowed RAM buffer, or the number diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java 2011-12-03 13:28:28.686203600 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java 2011-12-08 19:31:48.552819500 +0100 @@ -79,7 +79,7 @@ final InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, true, new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null)); - final NormsWriter normsWriter = new NormsWriter(); + final NormsConsumer normsWriter = new NormsConsumer(documentsWriterPerThread); final DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter); return new DocFieldProcessor(documentsWriterPerThread, docInverter); } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/FieldInfos.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/FieldInfos.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/FieldInfos.java 2011-12-03 13:28:28.580197500 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/FieldInfos.java 2011-12-08 19:31:47.726772200 +0100 @@ -407,7 +407,7 @@ public boolean hasNorms() { for (FieldInfo fi : this) { - if (!fi.omitNorms) { + if (fi.isIndexed && !fi.omitNorms) { return true; } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java 2011-12-04 20:33:24.855287300 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java 2011-12-08 19:31:48.817834600 +0100 @@ -277,8 +277,6 @@ /** *

Construct a FilterIndexReader based on the specified base reader. - * Directory locking for delete, undeleteAll, and setNorm operations is - * left to the base reader.

*

Note that base reader is closed if this FilterIndexReader is closed.

* @param in specified base reader. */ @@ -332,9 +330,6 @@ } @Override - protected void doUndeleteAll() throws CorruptIndexException, IOException {in.undeleteAll();} - - @Override public boolean hasNorms(String field) throws IOException { ensureOpen(); return in.hasNorms(field); @@ -347,29 +342,10 @@ } @Override - protected void doSetNorm(int d, String f, byte b) throws CorruptIndexException, IOException { - in.setNorm(d, f, b); - } - - @Override - public int docFreq(Term t) throws IOException { - ensureOpen(); - return in.docFreq(t); - } - - @Override public int docFreq(String field, BytesRef t) throws IOException { ensureOpen(); return in.docFreq(field, t); } - - @Override - protected void doDelete(int n) throws CorruptIndexException, IOException { in.deleteDocument(n); } - - @Override - protected void doCommit(Map commitUserData) throws IOException { - in.commit(commitUserData); - } @Override protected void doClose() throws IOException { @@ -450,4 +426,14 @@ ensureOpen(); return in.perDocValues(); } + + @Override + public IndexCommit getIndexCommit() throws IOException { + return in.getIndexCommit(); + } + + @Override + public int getTermInfosIndexDivisor() { + return in.getTermInfosIndexDivisor(); + } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/IndexFileNames.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/IndexFileNames.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/IndexFileNames.java 2011-12-01 13:37:57.334758500 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/IndexFileNames.java 2011-12-08 19:31:48.304805300 +0100 @@ -17,8 +17,6 @@ * limitations under the License. */ -import java.util.regex.Pattern; - import org.apache.lucene.index.codecs.Codec; // for javadocs // TODO: put all files under codec and remove all the static extensions here @@ -49,9 +47,6 @@ /** Name of the generation reference file name */ public static final String SEGMENTS_GEN = "segments." + GEN_EXTENSION; - - /** Extension of norms file */ - public static final String NORMS_EXTENSION = "nrm"; /** Extension of compound file */ public static final String COMPOUND_FILE_EXTENSION = "cfs"; @@ -65,9 +60,6 @@ /** Extension of deletes */ public static final String DELETES_EXTENSION = "del"; - /** Extension of separate norms */ - public static final String SEPARATE_NORMS_EXTENSION = "s"; - /** * This array contains all filename extensions used by * Lucene's index files, with one exception, namely the @@ -80,14 +72,9 @@ COMPOUND_FILE_ENTRIES_EXTENSION, DELETES_EXTENSION, GEN_EXTENSION, - NORMS_EXTENSION, COMPOUND_FILE_STORE_EXTENSION, }; - public static final String[] NON_STORE_INDEX_EXTENSIONS = new String[] { - NORMS_EXTENSION - }; - /** * Computes the full file name from base, extension and generation. If the * generation is -1, the file name is null. If it's 0, the file name is @@ -188,17 +175,5 @@ filename = filename.substring(0, idx); } return filename; - } - - /** - * Returns true if the given filename ends with the separate norms file - * pattern: {@code SEPARATE_NORMS_EXTENSION + "[0-9]+"}. - */ - public static boolean isSeparateNormsFile(String filename) { - int idx = filename.lastIndexOf('.'); - if (idx == -1) return false; - String ext = filename.substring(idx + 1); - return Pattern.matches(SEPARATE_NORMS_EXTENSION + "[0-9]+", ext); - } - + } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/IndexReader.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/IndexReader.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/IndexReader.java 2011-12-05 22:00:28.676922700 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/IndexReader.java 2011-12-08 19:31:50.355922600 +0100 @@ -32,7 +32,6 @@ import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.search.FieldCache; // javadocs import org.apache.lucene.search.SearcherManager; // javadocs -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.*; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; @@ -45,7 +44,7 @@

Concrete subclasses of IndexReader are usually constructed with a call to one of the static open() methods, e.g. {@link - #open(Directory, boolean)}. + #open(Directory)}.

For efficiency, in this API documents are often referred to via document numbers, non-negative integers which each name a unique @@ -53,9 +52,6 @@ as documents are added to and deleted from an index. Clients should thus not rely on a given document having the same number between sessions. -

An IndexReader can be opened on a directory for which an IndexWriter is - opened already, but it cannot be used to delete documents from the index then. -

NOTE: for backwards API compatibility, several methods are not listed as abstract, but have no useful implementations in this base class and @@ -66,13 +62,6 @@

- NOTE: as of 2.4, it's possible to open a read-only - IndexReader using the static open methods that accept the - boolean readOnly parameter. Such a reader may have better - concurrency. You must specify false if you want to - make changes with the resulting IndexReader. -

-

NOTE: {@link IndexReader} instances are completely thread safe, meaning multiple threads can call any of its methods, @@ -175,14 +164,13 @@ } private volatile boolean closed; - protected boolean hasChanges; private final AtomicInteger refCount = new AtomicInteger(); static int DEFAULT_TERMS_INDEX_DIVISOR = 1; /** Expert: returns the current refCount for this reader */ - public int getRefCount() { + public final int getRefCount() { return refCount.get(); } @@ -201,7 +189,7 @@ * @see #decRef * @see #tryIncRef */ - public void incRef() { + public final void incRef() { ensureOpen(); refCount.incrementAndGet(); } @@ -229,7 +217,7 @@ * @see #decRef * @see #incRef */ - public boolean tryIncRef() { + public final boolean tryIncRef() { int count; while ((count = refCount.get()) > 0) { if (refCount.compareAndSet(count, count+1)) { @@ -243,9 +231,6 @@ @Override public String toString() { final StringBuilder buffer = new StringBuilder(); - if (hasChanges) { - buffer.append('*'); - } buffer.append(getClass().getSimpleName()); buffer.append('('); final IndexReader[] subReaders = getSequentialSubReaders(); @@ -270,13 +255,12 @@ * * @see #incRef */ - public void decRef() throws IOException { + public final void decRef() throws IOException { ensureOpen(); final int rc = refCount.getAndDecrement(); if (rc == 1) { boolean success = false; try { - commit(); doClose(); success = true; } finally { @@ -305,27 +289,33 @@ } /** Returns a IndexReader reading the index in the given - * Directory, with readOnly=true. + * Directory * @param directory the index directory * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory) throws CorruptIndexException, IOException { - return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR); + return open(directory, null, DEFAULT_TERMS_INDEX_DIVISOR); } - - /** Returns an IndexReader reading the index in the given - * Directory. You should pass readOnly=true, since it - * gives much better concurrent performance, unless you - * intend to do write operations (delete documents or - * change norms) with the reader. + + /** Returns a IndexReader reading the index in the given + * Directory * @param directory the index directory - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader + * @param termInfosIndexDivisor Subsamples which indexed + * terms are loaded into RAM. This has the same effect as {@link + * IndexWriterConfig#setTermIndexInterval} except that setting + * must be done at indexing time while this setting can be + * set per reader. When set to N, then one in every + * N*termIndexInterval terms in the index is loaded into + * memory. By setting this to a value > 1 you can reduce + * memory usage, at the expense of higher latency when + * loading a TermInfo. The default value is 1. Set this + * to -1 to skip loading the terms index entirely. * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static IndexReader open(final Directory directory, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + public static IndexReader open(final Directory directory, int termInfosIndexDivisor) throws CorruptIndexException, IOException { + return open(directory, null, termInfosIndexDivisor); } /** @@ -352,48 +342,20 @@ } /** Expert: returns an IndexReader reading the index in the given - * {@link IndexCommit}. You should pass readOnly=true, since it - * gives much better concurrent performance, unless you - * intend to do write operations (delete documents or - * change norms) with the reader. + * {@link IndexCommit}. * @param commit the commit point to open - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static IndexReader open(final IndexCommit commit, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + public static IndexReader open(final IndexCommit commit) throws CorruptIndexException, IOException { + return open(commit.getDirectory(), commit, DEFAULT_TERMS_INDEX_DIVISOR); } - /** Expert: returns an IndexReader reading the index in - * the given Directory, with a custom {@link - * IndexDeletionPolicy}. You should pass readOnly=true, - * since it gives much better concurrent performance, - * unless you intend to do write operations (delete - * documents or change norms) with the reader. - * @param directory the index directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); - } - /** Expert: returns an IndexReader reading the index in - * the given Directory, with a custom {@link - * IndexDeletionPolicy}. You should pass readOnly=true, - * since it gives much better concurrent performance, - * unless you intend to do write operations (delete - * documents or change norms) with the reader. + /** Expert: returns an IndexReader reading the index in the given + * {@link IndexCommit}. * @param directory the index directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader + * @param commit the commit point to open * @param termInfosIndexDivisor Subsamples which indexed * terms are loaded into RAM. This has the same effect as {@link * IndexWriterConfig#setTermIndexInterval} except that setting @@ -407,65 +369,8 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor); - } - - /** Expert: returns an IndexReader reading the index in - * the given Directory, using a specific commit and with - * a custom {@link IndexDeletionPolicy}. You should pass - * readOnly=true, since it gives much better concurrent - * performance, unless you intend to do write operations - * (delete documents or change norms) with the reader. - * @param commit the specific {@link IndexCommit} to open; - * see {@link IndexReader#listCommits} to list all commits - * in a directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); - } - - /** Expert: returns an IndexReader reading the index in - * the given Directory, using a specific commit and with - * a custom {@link IndexDeletionPolicy}. You should pass - * readOnly=true, since it gives much better concurrent - * performance, unless you intend to do write operations - * (delete documents or change norms) with the reader. - * @param commit the specific {@link IndexCommit} to open; - * see {@link IndexReader#listCommits} to list all commits - * in a directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader - * @param termInfosIndexDivisor Subsamples which indexed - * terms are loaded into RAM. This has the same effect as {@link - * IndexWriterConfig#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1. Set this - * to -1 to skip loading the terms index entirely. This is only useful in - * advanced situations when you will only .next() through all terms; - * attempts to seek will hit an exception. - * - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor); - } - - private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor); + public static IndexReader open(final Directory directory, final IndexCommit commit, int termInfosIndexDivisor) throws CorruptIndexException, IOException { + return DirectoryReader.open(directory, commit, termInfosIndexDivisor); } /** @@ -487,11 +392,6 @@ * reader while other threads are still using it; see * {@link SearcherManager} to simplify managing this. * - *

If a new reader is returned, it's safe to make changes - * (deletions, norms) with it. All shared mutable state - * with the old reader uses "copy on write" semantics to - * ensure the changes are not seen by other readers. - * * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error * @return null if there are no changes; else, a new @@ -504,28 +404,12 @@ } /** - * If the index has changed since the provided reader was - * opened, open and return a new reader, with the - * specified readOnly; else, return - * null. - * - * @see #openIfChanged(IndexReader) - */ - public static IndexReader openIfChanged(IndexReader oldReader, boolean readOnly) throws IOException { - final IndexReader newReader = oldReader.doOpenIfChanged(readOnly); - assert newReader != oldReader; - return newReader; - } - - /** * If the IndexCommit differs from what the - * provided reader is searching, or the provided reader is - * not already read-only, open and return a new - * readOnly=true reader; else, return null. + * provided reader is searching, open and return a new + * reader; else, return null. * * @see #openIfChanged(IndexReader) */ - // TODO: should you be able to specify readOnly? public static IndexReader openIfChanged(IndexReader oldReader, IndexCommit commit) throws IOException { final IndexReader newReader = oldReader.doOpenIfChanged(commit); assert newReader != oldReader; @@ -535,7 +419,7 @@ /** * Expert: If there changes (committed or not) in the * {@link IndexWriter} versus what the provided reader is - * searching, then open and return a new read-only + * searching, then open and return a new * IndexReader searching both committed and uncommitted * changes from the writer; else, return null (though, the * current implementation never returns null). @@ -613,16 +497,6 @@ * If the index has changed since it was opened, open and return a new reader; * else, return {@code null}. * - * @see #openIfChanged(IndexReader, boolean) - */ - protected IndexReader doOpenIfChanged(boolean openReadOnly) throws CorruptIndexException, IOException { - throw new UnsupportedOperationException("This reader does not support reopen()."); - } - - /** - * If the index has changed since it was opened, open and return a new reader; - * else, return {@code null}. - * * @see #openIfChanged(IndexReader, IndexCommit) */ protected IndexReader doOpenIfChanged(final IndexCommit commit) throws CorruptIndexException, IOException { @@ -642,34 +516,11 @@ /** * Efficiently clones the IndexReader (sharing most * internal state). - *

- * On cloning a reader with pending changes (deletions, - * norms), the original reader transfers its write lock to - * the cloned reader. This means only the cloned reader - * may make further changes to the index, and commit the - * changes to the index on close, but the old reader still - * reflects all changes made up until it was cloned. - *

- * Like {@link #openIfChanged(IndexReader)}, it's safe to make changes to - * either the original or the cloned reader: all shared - * mutable state obeys "copy on write" semantics to ensure - * the changes are not seen by other readers. - *

*/ @Override public synchronized Object clone() { throw new UnsupportedOperationException("This reader does not implement clone()"); } - - /** - * Clones the IndexReader and optionally changes readOnly. A readOnly - * reader cannot open a writeable reader. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException { - throw new UnsupportedOperationException("This reader does not implement clone()"); - } /** * Returns the directory associated with this index. The Default @@ -807,14 +658,14 @@ * term vectors were not indexed. The returned Fields * instance acts like a single-document inverted index * (the docID will be 0). */ - abstract public Fields getTermVectors(int docID) + public abstract Fields getTermVectors(int docID) throws IOException; /** Retrieve term vector for this document and field, or * null if term vectors were not indexed. The returned * Fields instance acts like a single-document inverted * index (the docID will be 0). */ - public Terms getTermVector(int docID, String field) + public final Terms getTermVector(int docID, String field) throws IOException { Fields vectors = getTermVectors(docID); if (vectors == null) { @@ -848,7 +699,7 @@ public abstract int maxDoc(); /** Returns the number of deleted documents. */ - public int numDeletedDocs() { + public final int numDeletedDocs() { return maxDoc() - numDocs(); } @@ -881,7 +732,7 @@ // TODO: we need a separate StoredField, so that the // Document returned here contains that class not // IndexableField - public Document document(int docID) throws CorruptIndexException, IOException { + public final Document document(int docID) throws CorruptIndexException, IOException { ensureOpen(); if (docID < 0 || docID >= maxDoc()) { throw new IllegalArgumentException("docID must be >= 0 and < maxDoc=" + maxDoc() + " (got docID=" + docID + ")"); @@ -910,39 +761,6 @@ */ public abstract byte[] norms(String field) throws IOException; - /** Expert: Resets the normalization factor for the named field of the named - * document. By default, the norm represents the product of the field's {@link - * org.apache.lucene.document.Field#setBoost(float) boost} and its - * length normalization}. Thus, to preserve the length normalization - * values when resetting this, one should base the new value upon the old. - * - * NOTE: If this field does not index norms, then - * this method throws {@link IllegalStateException}. - * - * @see #norms(String) - * @see Similarity#computeNorm(FieldInvertState) - * @see org.apache.lucene.search.similarities.DefaultSimilarity#decodeNormValue(byte) - * @throws StaleReaderException if the index has changed - * since this reader was opened - * @throws CorruptIndexException if the index is corrupt - * @throws LockObtainFailedException if another writer - * has this index open (write.lock could not - * be obtained) - * @throws IOException if there is a low-level IO error - * @throws IllegalStateException if the field does not index norms - */ - public synchronized void setNorm(int doc, String field, byte value) - throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException { - ensureOpen(); - acquireWriteLock(); - hasChanges = true; - doSetNorm(doc, field, value); - } - - /** Implements setNorm in subclass.*/ - protected abstract void doSetNorm(int doc, String field, byte value) - throws CorruptIndexException, IOException; - /** * Returns {@link Fields} for this reader. * This method may return null if the reader has no @@ -973,7 +791,7 @@ * through them yourself. */ public abstract PerDocValues perDocValues() throws IOException; - public int docFreq(Term term) throws IOException { + public final int docFreq(Term term) throws IOException { return docFreq(term.field(), term.bytes()); } @@ -1004,7 +822,7 @@ * field does not exists. This method does not take into * account deleted documents that have not yet been merged * away. */ - public long totalTermFreq(String field, BytesRef term) throws IOException { + public final long totalTermFreq(String field, BytesRef term) throws IOException { final Fields fields = fields(); if (fields == null) { return 0; @@ -1022,7 +840,7 @@ } /** This may return null if the field does not exist.*/ - public Terms terms(String field) throws IOException { + public final Terms terms(String field) throws IOException { final Fields fields = fields(); if (fields == null) { return null; @@ -1033,7 +851,7 @@ /** Returns {@link DocsEnum} for the specified field & * term. This may return null, if either the field or * term does not exist. */ - public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException { + public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException { assert field != null; assert term != null; final Fields fields = fields(); @@ -1053,7 +871,7 @@ * field & term. This may return null, if either the * field or term does not exist, or, positions were not * indexed for this field. */ - public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException { + public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException { assert field != null; assert term != null; final Fields fields = fields(); @@ -1074,7 +892,7 @@ * {@link TermState}. This may return null, if either the field or the term * does not exists or the {@link TermState} is invalid for the underlying * implementation.*/ - public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state, boolean needsFreqs) throws IOException { + public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state, boolean needsFreqs) throws IOException { assert state != null; assert field != null; final Fields fields = fields(); @@ -1094,7 +912,7 @@ * {@link TermState}. This may return null, if either the field or the term * does not exists, the {@link TermState} is invalid for the underlying * implementation, or positions were not indexed for this field. */ - public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException { + public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException { assert state != null; assert field != null; final Fields fields = fields(); @@ -1109,159 +927,6 @@ return null; } - - /** Deletes the document numbered docNum. Once a document is - * deleted it will not appear in TermDocs or TermPositions enumerations. - * Attempts to read its field with the {@link #document} - * method will result in an error. The presence of this document may still be - * reflected in the {@link #docFreq} statistic, though - * this will be corrected eventually as the index is further modified. - * - * @throws StaleReaderException if the index has changed - * since this reader was opened - * @throws CorruptIndexException if the index is corrupt - * @throws LockObtainFailedException if another writer - * has this index open (write.lock could not - * be obtained) - * @throws IOException if there is a low-level IO error - */ - public synchronized void deleteDocument(int docNum) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException { - ensureOpen(); - acquireWriteLock(); - hasChanges = true; - doDelete(docNum); - } - - - /** Implements deletion of the document numbered docNum. - * Applications should call {@link #deleteDocument(int)} or {@link #deleteDocuments(Term)}. - */ - protected abstract void doDelete(int docNum) throws CorruptIndexException, IOException; - - - /** Deletes all documents that have a given term indexed. - * This is useful if one uses a document field to hold a unique ID string for - * the document. Then to delete such a document, one merely constructs a - * term with the appropriate field and the unique ID string as its text and - * passes it to this method. - * See {@link #deleteDocument(int)} for information about when this deletion will - * become effective. - * - * @return the number of documents deleted - * @throws StaleReaderException if the index has changed - * since this reader was opened - * @throws CorruptIndexException if the index is corrupt - * @throws LockObtainFailedException if another writer - * has this index open (write.lock could not - * be obtained) - * @throws IOException if there is a low-level IO error - */ - public int deleteDocuments(Term term) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException { - ensureOpen(); - DocsEnum docs = MultiFields.getTermDocsEnum(this, - MultiFields.getLiveDocs(this), - term.field(), - term.bytes(), - false); - if (docs == null) { - return 0; - } - int n = 0; - int doc; - while ((doc = docs.nextDoc()) != DocsEnum.NO_MORE_DOCS) { - deleteDocument(doc); - n++; - } - return n; - } - - /** Undeletes all documents currently marked as deleted in - * this index. - * - *

NOTE: this method can only recover documents marked - * for deletion but not yet removed from the index; when - * and how Lucene removes deleted documents is an - * implementation detail, subject to change from release - * to release. However, you can use {@link - * #numDeletedDocs} on the current IndexReader instance to - * see how many documents will be un-deleted. - * - * @throws StaleReaderException if the index has changed - * since this reader was opened - * @throws LockObtainFailedException if another writer - * has this index open (write.lock could not - * be obtained) - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public synchronized void undeleteAll() throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException { - ensureOpen(); - acquireWriteLock(); - hasChanges = true; - doUndeleteAll(); - } - - /** Implements actual undeleteAll() in subclass. */ - protected abstract void doUndeleteAll() throws CorruptIndexException, IOException; - - /** Does nothing by default. Subclasses that require a write lock for - * index modifications must implement this method. */ - protected synchronized void acquireWriteLock() throws IOException { - /* NOOP */ - } - - /** - * - * @throws IOException - */ - public final synchronized void flush() throws IOException { - ensureOpen(); - commit(); - } - - /** - * @param commitUserData Opaque Map (String -> String) - * that's recorded into the segments file in the index, - * and retrievable by {@link - * IndexReader#getCommitUserData}. - * @throws IOException - */ - public final synchronized void flush(Map commitUserData) throws IOException { - ensureOpen(); - commit(commitUserData); - } - - /** - * Commit changes resulting from delete, undeleteAll, or - * setNorm operations - * - * If an exception is hit, then either no changes or all - * changes will have been committed to the index - * (transactional semantics). - * @throws IOException if there is a low-level IO error - */ - protected final synchronized void commit() throws IOException { - commit(null); - } - - /** - * Commit changes resulting from delete, undeleteAll, or - * setNorm operations - * - * If an exception is hit, then either no changes or all - * changes will have been committed to the index - * (transactional semantics). - * @throws IOException if there is a low-level IO error - */ - public final synchronized void commit(Map commitUserData) throws IOException { - // Don't call ensureOpen since we commit() on close - doCommit(commitUserData); - hasChanges = false; - } - - /** Implements commit. */ - protected abstract void doCommit(Map commitUserData) throws IOException; - /** * Closes files associated with this index. * Also saves any new deletions to disk. @@ -1397,7 +1062,7 @@ * one commit point. But if you're using a custom {@link * IndexDeletionPolicy} then there could be many commits. * Once you have a given commit, you can open a reader on - * it by calling {@link IndexReader#open(IndexCommit,boolean)} + * it by calling {@link IndexReader#open(IndexCommit)} * There must be at least one commit in * the Directory, else this method throws {@link * IndexNotFoundException}. Note that if a commit is in @@ -1418,7 +1083,7 @@ * that has no sub readers). *

* NOTE: You should not try using sub-readers returned by - * this method to make any changes (setNorm, deleteDocument, + * this method to make any changes (deleteDocument, * etc.). While this might succeed for one composite reader * (like MultiReader), it will most likely lead to index * corruption for other readers (like DirectoryReader obtained @@ -1444,12 +1109,6 @@ * top-level context holds a null {@link CompositeReaderContext#leaves} * reference. Only the top-level context maintains the convenience leaf-view * for performance reasons. - *

- * NOTE: You should not try using sub-readers returned by this method to make - * any changes (setNorm, deleteDocument, etc.). While this might succeed for - * one composite reader (like MultiReader), it will most likely lead to index - * corruption for other readers (like DirectoryReader obtained through - * {@link #open}. Use the top-level context's reader directly. * * @lucene.experimental */ @@ -1470,7 +1129,10 @@ * Instead, you should call {@link * #getSequentialSubReaders} and ask each sub reader for * its unique term count. */ - public long getUniqueTermCount() throws IOException { + public final long getUniqueTermCount() throws IOException { + if (!getTopReaderContext().isAtomic) { + return -1; + } final Fields fields = fields(); if (fields == null) { return 0; diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/IndexWriter.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/IndexWriter.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/IndexWriter.java 2011-12-03 13:28:28.892215300 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/IndexWriter.java 2011-12-08 19:31:50.889953100 +0100 @@ -30,6 +30,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; @@ -618,7 +619,7 @@ SegmentReader sr = ent.getValue(); if (sr.hasChanges) { assert infoIsLive(sr.getSegmentInfo(), "key=" + ent.getKey()); - sr.doCommit(null); + sr.doCommit(); // Must checkpoint w/ deleter, because this // segment reader will have created new _X_N.del @@ -650,7 +651,7 @@ final SegmentReader sr = readerMap.get(new SegmentCacheKey(info, IOContext.Context.READ)); if (sr != null && sr.hasChanges) { assert infoIsLive(info); - sr.doCommit(null); + sr.doCommit(); // Must checkpoint w/ deleter, because this // segment reader will have created new _X_N.del // file. @@ -697,7 +698,7 @@ // TODO: we may want to avoid doing this while // synchronized // Returns a ref, which we xfer to readerMap: - sr = SegmentReader.get(false, info.dir, info, doOpenStores, context.context == IOContext.Context.MERGE ? -1 : config.getReaderTermsIndexDivisor(), context); + sr = SegmentReader.getRW(info, doOpenStores, context.context == IOContext.Context.MERGE ? -1 : config.getReaderTermsIndexDivisor(), context); sr.readerFinishedListeners = readerFinishedListeners; if (info.dir == directory) { @@ -3980,7 +3981,7 @@ for (String file : files) { assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION) : ".del file is not allowed in .cfs: " + file; - assert !IndexFileNames.isSeparateNormsFile(file) + assert !isSeparateNormsFile(file) : "separate norms file (.s[0-9]+) is not allowed in .cfs: " + file; directory.copy(cfsDir, file, file, context); checkAbort.work(directory.fileLength(file)); @@ -3991,4 +3992,18 @@ return files; } + + + /** + * Returns true if the given filename ends with the separate norms file + * pattern: {@code SEPARATE_NORMS_EXTENSION + "[0-9]+"}. + * @deprecated only for asserting + */ + @Deprecated + private static boolean isSeparateNormsFile(String filename) { + int idx = filename.lastIndexOf('.'); + if (idx == -1) return false; + String ext = filename.substring(idx + 1); + return Pattern.matches("s[0-9]+", ext); + } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/MultiReader.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/MultiReader.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/MultiReader.java 2011-11-16 23:30:34.365992900 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/MultiReader.java 2011-12-08 19:31:47.429755200 +0100 @@ -41,8 +41,6 @@ /** *

Construct a MultiReader aggregating the named set of (sub)readers. - * Directory locking for delete, undeleteAll, and setNorm operations is - * left to the subreaders.

*

Note that all subreaders are closed if this Multireader is closed.

* @param subReaders set of (sub)readers */ @@ -52,8 +50,6 @@ /** *

Construct a MultiReader aggregating the named set of (sub)readers. - * Directory locking for delete, undeleteAll, and setNorm operations is - * left to the subreaders.

* @param closeSubReaders indicates whether the subreaders should be closed * when this MultiReader is closed * @param subReaders set of (sub)readers @@ -87,11 +83,6 @@ } @Override - public long getUniqueTermCount() throws IOException { - throw new UnsupportedOperationException(""); - } - - @Override public Fields fields() throws IOException { throw new UnsupportedOperationException("please use MultiFields.getFields, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Fields"); } @@ -243,23 +234,6 @@ return hasDeletions; } - @Override - protected void doDelete(int n) throws CorruptIndexException, IOException { - numDocs = -1; // invalidate cache - int i = readerIndex(n); // find segment num - subReaders[i].deleteDocument(n - starts[i]); // dispatch to segment reader - hasDeletions = true; - } - - @Override - protected void doUndeleteAll() throws CorruptIndexException, IOException { - for (int i = 0; i < subReaders.length; i++) - subReaders[i].undeleteAll(); - - hasDeletions = false; - numDocs = -1; // invalidate cache - } - private int readerIndex(int n) { // find reader for doc n: return DirectoryReader.readerIndex(n, this.starts, this.subReaders.length); } @@ -277,22 +251,6 @@ public synchronized byte[] norms(String field) throws IOException { throw new UnsupportedOperationException("please use MultiNorms.norms, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level norms"); } - - @Override - protected void doSetNorm(int n, String field, byte value) - throws CorruptIndexException, IOException { - int i = readerIndex(n); // find segment num - subReaders[i].setNorm(n-starts[i], field, value); // dispatch - } - - @Override - public int docFreq(Term t) throws IOException { - ensureOpen(); - int total = 0; // sum freqs in segments - for (int i = 0; i < subReaders.length; i++) - total += subReaders[i].docFreq(t); - return total; - } @Override public int docFreq(String field, BytesRef t) throws IOException { @@ -303,12 +261,6 @@ } return total; } - - @Override - protected void doCommit(Map commitUserData) throws IOException { - for (int i = 0; i < subReaders.length; i++) - subReaders[i].commit(commitUserData); - } @Override protected synchronized void doClose() throws IOException { diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/NormsConsumer.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/NormsConsumer.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/NormsConsumer.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/NormsConsumer.java 2011-12-08 19:31:51.153968200 +0100 @@ -0,0 +1,111 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +import org.apache.lucene.index.codecs.NormsFormat; +import org.apache.lucene.index.codecs.NormsWriter; +import org.apache.lucene.util.IOUtils; + +// TODO FI: norms could actually be stored as doc store + +/** Writes norms. Each thread X field accumulates the norms + * for the doc/fields it saw, then the flush method below + * merges all of these together into a single _X.nrm file. + */ + +final class NormsConsumer extends InvertedDocEndConsumer { + final NormsFormat normsFormat; + + public NormsConsumer(DocumentsWriterPerThread dwpt) { + normsFormat = dwpt.codec.normsFormat(); + } + + @Override + public void abort() {} + + // We only write the _X.nrm file at flush + void files(Collection files) {} + + /** Produce _X.nrm if any document had a field with norms + * not disabled */ + @Override + public void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException { + if (!state.fieldInfos.hasNorms()) { + return; + } + + NormsWriter normsOut = null; + boolean success = false; + try { + normsOut = normsFormat.normsWriter(state); + + for (FieldInfo fi : state.fieldInfos) { + final NormsConsumerPerField toWrite = (NormsConsumerPerField) fieldsToFlush.get(fi); + int upto = 0; + // we must check the final value of omitNorms for the fieldinfo, it could have + // changed for this field since the first time we added it. + if (!fi.omitNorms && toWrite != null && toWrite.upto > 0) { + normsOut.startField(fi); + int docID = 0; + for (; docID < state.numDocs; docID++) { + if (upto < toWrite.upto && toWrite.docIDs[upto] == docID) { + normsOut.writeNorm(toWrite.norms[upto]); + upto++; + } else { + normsOut.writeNorm((byte) 0); + } + } + + // we should have consumed every norm + assert upto == toWrite.upto; + + toWrite.reset(); + } else if (fi.isIndexed && !fi.omitNorms) { + // Fill entire field with default norm: + normsOut.startField(fi); + for(;upto { + + final FieldInfo fieldInfo; + final DocumentsWriterPerThread.DocState docState; + final Similarity similarity; + + // Holds all docID/norm pairs we've seen + int[] docIDs = new int[1]; + byte[] norms = new byte[1]; + int upto; + + final FieldInvertState fieldState; + + public void reset() { + // Shrink back if we are overallocated now: + docIDs = ArrayUtil.shrink(docIDs, upto); + norms = ArrayUtil.shrink(norms, upto); + upto = 0; + } + + public NormsConsumerPerField(final DocInverterPerField docInverterPerField, final FieldInfo fieldInfo) { + this.fieldInfo = fieldInfo; + docState = docInverterPerField.docState; + fieldState = docInverterPerField.fieldState; + similarity = docState.similarityProvider.get(fieldInfo.name); + } + + @Override + void abort() { + upto = 0; + } + + public int compareTo(NormsConsumerPerField other) { + return fieldInfo.name.compareTo(other.fieldInfo.name); + } + + @Override + void finish() { + if (fieldInfo.isIndexed && !fieldInfo.omitNorms) { + if (docIDs.length <= upto) { + assert docIDs.length == upto; + docIDs = ArrayUtil.grow(docIDs, 1+upto); + } + if (norms.length <= upto) { + assert norms.length == upto; + norms = ArrayUtil.grow(norms, 1+upto); + } + norms[upto] = similarity.computeNorm(fieldState); + docIDs[upto] = docState.docID; + upto++; + } + } +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/NormsWriter.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/NormsWriter.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/NormsWriter.java 2011-12-05 22:00:28.681923000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/NormsWriter.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,112 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Collection; -import java.util.Map; - -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IOContext.Context; -import org.apache.lucene.util.IOUtils; - -// TODO FI: norms could actually be stored as doc store - -/** Writes norms. Each thread X field accumulates the norms - * for the doc/fields it saw, then the flush method below - * merges all of these together into a single _X.nrm file. - */ - -final class NormsWriter extends InvertedDocEndConsumer { - - - @Override - public void abort() {} - - // We only write the _X.nrm file at flush - void files(Collection files) {} - - /** Produce _X.nrm if any document had a field with norms - * not disabled */ - @Override - public void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException { - if (!state.fieldInfos.hasNorms()) { - return; - } - - final String normsFileName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.NORMS_EXTENSION); - IndexOutput normsOut = state.directory.createOutput(normsFileName, state.context); - boolean success = false; - try { - normsOut.writeBytes(SegmentNorms.NORMS_HEADER, 0, SegmentNorms.NORMS_HEADER.length); - - int normCount = 0; - - for (FieldInfo fi : state.fieldInfos) { - final NormsWriterPerField toWrite = (NormsWriterPerField) fieldsToFlush.get(fi); - int upto = 0; - // we must check the final value of omitNorms for the fieldinfo, it could have - // changed for this field since the first time we added it. - if (!fi.omitNorms && toWrite != null && toWrite.upto > 0) { - normCount++; - - int docID = 0; - for (; docID < state.numDocs; docID++) { - if (upto < toWrite.upto && toWrite.docIDs[upto] == docID) { - normsOut.writeByte(toWrite.norms[upto]); - upto++; - } else { - normsOut.writeByte((byte) 0); - } - } - - // we should have consumed every norm - assert upto == toWrite.upto; - - toWrite.reset(); - } else if (fi.isIndexed && !fi.omitNorms) { - normCount++; - // Fill entire field with default norm: - for(;upto { - - final FieldInfo fieldInfo; - final DocumentsWriterPerThread.DocState docState; - final Similarity similarity; - - // Holds all docID/norm pairs we've seen - int[] docIDs = new int[1]; - byte[] norms = new byte[1]; - int upto; - - final FieldInvertState fieldState; - - public void reset() { - // Shrink back if we are overallocated now: - docIDs = ArrayUtil.shrink(docIDs, upto); - norms = ArrayUtil.shrink(norms, upto); - upto = 0; - } - - public NormsWriterPerField(final DocInverterPerField docInverterPerField, final FieldInfo fieldInfo) { - this.fieldInfo = fieldInfo; - docState = docInverterPerField.docState; - fieldState = docInverterPerField.fieldState; - similarity = docState.similarityProvider.get(fieldInfo.name); - } - - @Override - void abort() { - upto = 0; - } - - public int compareTo(NormsWriterPerField other) { - return fieldInfo.name.compareTo(other.fieldInfo.name); - } - - @Override - void finish() { - if (fieldInfo.isIndexed && !fieldInfo.omitNorms) { - if (docIDs.length <= upto) { - assert docIDs.length == upto; - docIDs = ArrayUtil.grow(docIDs, 1+upto); - } - if (norms.length <= upto) { - assert norms.length == upto; - norms = ArrayUtil.grow(norms, 1+upto); - } - norms[upto] = similarity.computeNorm(fieldState); - docIDs[upto] = docState.docID; - upto++; - } - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/ParallelReader.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/ParallelReader.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/ParallelReader.java 2011-11-16 23:30:34.570004600 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/ParallelReader.java 2011-12-08 19:31:49.691884600 +0100 @@ -332,24 +332,6 @@ return hasDeletions; } - // delete in all readers - @Override - protected void doDelete(int n) throws CorruptIndexException, IOException { - for (final IndexReader reader : readers) { - reader.deleteDocument(n); - } - hasDeletions = true; - } - - // undeleteAll in all readers - @Override - protected void doUndeleteAll() throws CorruptIndexException, IOException { - for (final IndexReader reader : readers) { - reader.undeleteAll(); - } - hasDeletions = false; - } - @Override public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException { ensureOpen(); @@ -403,25 +385,6 @@ } @Override - protected void doSetNorm(int n, String field, byte value) - throws CorruptIndexException, IOException { - IndexReader reader = fieldToReader.get(field); - if (reader!=null) { - synchronized(normsCache) { - normsCache.remove(field); - } - reader.doSetNorm(n, field, value); - } - } - - @Override - public int docFreq(Term term) throws IOException { - ensureOpen(); - IndexReader reader = fieldToReader.get(term.field()); - return reader==null ? 0 : reader.docFreq(term); - } - - @Override public int docFreq(String field, BytesRef term) throws IOException { ensureOpen(); IndexReader reader = fieldToReader.get(field); @@ -458,12 +421,6 @@ } @Override - protected void doCommit(Map commitUserData) throws IOException { - for (final IndexReader reader : readers) - reader.commit(commitUserData); - } - - @Override protected synchronized void doClose() throws IOException { for (int i = 0; i < readers.size(); i++) { if (decrefOnClose.get(i).booleanValue()) { diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java 2011-10-21 19:50:44.808229400 +0200 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java 2011-12-08 19:31:56.050248300 +0100 @@ -62,7 +62,7 @@ * keeps a lock on the snapshots directory). */ public static Map readSnapshotsInfo(Directory dir) throws IOException { - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); Map snapshots = new HashMap(); try { int numDocs = r.numDocs(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java 2011-11-16 23:30:36.756129600 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java 2011-12-08 19:31:55.739230500 +0100 @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.NormsReader; import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.StoredFieldsReader; @@ -48,6 +49,7 @@ final FieldsProducer fields; final PerDocValues perDocProducer; + final NormsReader norms; final Directory dir; final Directory cfsDir; @@ -92,6 +94,10 @@ // Ask codec for its Fields fields = format.fieldsProducer(segmentReadState); assert fields != null; + // ask codec for its Norms: + // TODO: since we don't write any norms file if there are no norms, + // kinda jaky to assume the codec handles the case of no norms file at all gracefully?! + norms = codec.normsFormat().normsReader(cfsDir, si, fieldInfos, context, dir); perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState); success = true; } finally { @@ -126,7 +132,7 @@ synchronized void decRef() throws IOException { if (ref.decrementAndGet() == 0) { IOUtils.close(fields, perDocProducer, termVectorsReaderOrig, - fieldsReaderOrig, cfsReader, storeCFSReader); + fieldsReaderOrig, cfsReader, storeCFSReader, norms); // Now, notify any ReaderFinished listeners: if (owner != null) { owner.notifyReaderFinishedListeners(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentInfo.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentInfo.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentInfo.java 2011-11-16 23:30:34.560004000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentInfo.java 2011-12-08 19:31:49.117851800 +0100 @@ -48,7 +48,7 @@ // TODO: remove these from this class, for now this is the representation public static final int NO = -1; // e.g. no norms; no deletes; public static final int YES = 1; // e.g. have norms; have deletes; - static final int WITHOUT_GEN = 0; // a file name that has no GEN in it. + public static final int WITHOUT_GEN = 0; // a file name that has no GEN in it. public String name; // unique name in dir public int docCount; // number of docs in seg @@ -337,23 +337,10 @@ } /** - * Returns true if this field for this segment has saved a separate norms file (__N.sX). - * - * @param fieldNumber the field index to check - */ - public boolean hasSeparateNorms(int fieldNumber) { - if (normGen == null) { - return false; - } - - Long gen = normGen.get(fieldNumber); - return gen != null && gen.longValue() != NO; - } - - /** - * Returns true if any fields in this segment have separate norms. + * @deprecated separate norms are not supported in >= 4.0 */ - public boolean hasSeparateNorms() { + @Deprecated + boolean hasSeparateNorms() { if (normGen == null) { return false; } else { @@ -367,42 +354,6 @@ return false; } - void initNormGen() { - if (normGen == null) { // normGen is null if this segments file hasn't had any norms set against it yet - normGen = new HashMap(); - } - } - - /** - * Increment the generation count for the norms file for - * this field. - * - * @param fieldIndex field whose norm file will be rewritten - */ - void advanceNormGen(int fieldIndex) { - Long gen = normGen.get(fieldIndex); - if (gen == null || gen.longValue() == NO) { - normGen.put(fieldIndex, new Long(YES)); - } else { - normGen.put(fieldIndex, gen+1); - } - clearFilesCache(); - } - - /** - * Get the file name for the norms file for this field. - * - * @param number field index - */ - public String getNormFileName(int number) { - if (hasSeparateNorms(number)) { - return IndexFileNames.fileNameFromGeneration(name, IndexFileNames.SEPARATE_NORMS_EXTENSION + number, normGen.get(number)); - } else { - // single file for all norms - return IndexFileNames.fileNameFromGeneration(name, IndexFileNames.NORMS_EXTENSION, WITHOUT_GEN); - } - } - /** * Mark whether this segment is stored as a compound file. * @@ -516,11 +467,6 @@ return codec; } - private void addIfExists(Set files, String fileName) throws IOException { - if (dir.fileExists(fileName)) - files.add(fileName); - } - /* * Return all files referenced by this SegmentInfo. The * returns List is a locally cached List so you should not @@ -546,9 +492,6 @@ IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); } } else { - for(String ext : IndexFileNames.NON_STORE_INDEX_EXTENSIONS) { - addIfExists(fileSet, IndexFileNames.segmentFileName(name, "", ext)); - } codec.files(dir, this, fileSet); } @@ -566,15 +509,12 @@ if (delFileName != null && (delGen >= YES || dir.fileExists(delFileName))) { fileSet.add(delFileName); } - + + // because separate norm files are unconditionally stored outside cfs, + // we must explicitly ask for their filenames if we might have separate norms: + // remove this when 3.x indexes are no longer supported if (normGen != null) { - for (Entry entry : normGen.entrySet()) { - long gen = entry.getValue(); - if (gen >= YES) { - // Definitely a separate norm file, with generation: - fileSet.add(IndexFileNames.fileNameFromGeneration(name, IndexFileNames.SEPARATE_NORMS_EXTENSION + entry.getKey(), gen)); - } - } + codec.normsFormat().separateFiles(dir, this, fileSet); } files = new ArrayList(fileSet); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentMerger.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentMerger.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentMerger.java 2011-12-08 00:53:39.808791900 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentMerger.java 2011-12-08 19:31:48.958842700 +0100 @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -28,13 +27,13 @@ import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldInfosWriter; import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.NormsWriter; import org.apache.lucene.index.codecs.StoredFieldsWriter; import org.apache.lucene.index.codecs.PerDocConsumer; import org.apache.lucene.index.codecs.TermVectorsWriter; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.Bits; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.InfoStream; @@ -122,7 +121,11 @@ final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, segment, mergeState.fieldInfos, mergeState.mergedDocCount, termIndexInterval, codec, null, context); mergeTerms(segmentWriteState); mergePerDoc(segmentWriteState); - mergeNorms(); + + if (mergeState.fieldInfos.hasNorms()) { + int numMerged = mergeNorms(segmentWriteState); + assert numMerged == mergeState.mergedDocCount; + } if (mergeState.fieldInfos.hasVectors()) { int numMerged = mergeVectors(); @@ -329,48 +332,19 @@ } } - private void mergeNorms() throws IOException { - IndexOutput output = null; + private int mergeNorms(SegmentWriteState segmentWriteState) throws IOException { + final NormsWriter writer = codec.normsFormat().normsWriter(segmentWriteState); + boolean success = false; try { - for (FieldInfo fi : mergeState.fieldInfos) { - if (fi.isIndexed && !fi.omitNorms) { - if (output == null) { - output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION), context); - output.writeBytes(SegmentNorms.NORMS_HEADER, SegmentNorms.NORMS_HEADER.length); - } - for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) { - final int maxDoc = reader.reader.maxDoc(); - byte normBuffer[] = reader.reader.norms(fi.name); - if (normBuffer == null) { - // Can be null if this segment doesn't have - // any docs with this field - normBuffer = new byte[maxDoc]; - Arrays.fill(normBuffer, (byte)0); - } - if (reader.liveDocs == null) { - //optimized case for segments without deleted docs - output.writeBytes(normBuffer, maxDoc); - } else { - // this segment has deleted docs, so we have to - // check for every doc if it is deleted or not - final Bits liveDocs = reader.liveDocs; - for (int k = 0; k < maxDoc; k++) { - if (liveDocs.get(k)) { - output.writeByte(normBuffer[k]); - } - } - } - mergeState.checkAbort.work(maxDoc); - } - } - } + int numMerged = writer.merge(mergeState); success = true; + return numMerged; } finally { if (success) { - IOUtils.close(output); + IOUtils.close(writer); } else { - IOUtils.closeWhileHandlingException(output); + IOUtils.closeWhileHandlingException(writer); } } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentNorms.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentNorms.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentNorms.java 2011-10-19 14:22:11.424238900 +0200 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentNorms.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,245 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.lucene.store.FlushInfo; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -/** - * Byte[] referencing is used because a new norm object needs - * to be created for each clone, and the byte array is all - * that is needed for sharing between cloned readers. The - * current norm referencing is for sharing between readers - * whereas the byte[] referencing is for copy on write which - * is independent of reader references (i.e. incRef, decRef). - */ - -final class SegmentNorms implements Cloneable { - - /** norms header placeholder */ - static final byte[] NORMS_HEADER = new byte[]{'N','R','M',-1}; - - int refCount = 1; - - // If this instance is a clone, the originalNorm - // references the Norm that has a real open IndexInput: - private SegmentNorms origNorm; - - private IndexInput in; - private long normSeek; - - // null until bytes is set - private AtomicInteger bytesRef; - private byte[] bytes; - private int number; - - boolean dirty; - boolean rollbackDirty; - - private final SegmentReader owner; - - public SegmentNorms(IndexInput in, int number, long normSeek, SegmentReader owner) { - this.in = in; - this.number = number; - this.normSeek = normSeek; - this.owner = owner; - } - - public synchronized void incRef() { - assert refCount > 0 && (origNorm == null || origNorm.refCount > 0); - refCount++; - } - - private void closeInput() throws IOException { - if (in != null) { - if (in != owner.singleNormStream) { - // It's private to us -- just close it - in.close(); - } else { - // We are sharing this with others -- decRef and - // maybe close the shared norm stream - if (owner.singleNormRef.decrementAndGet() == 0) { - owner.singleNormStream.close(); - owner.singleNormStream = null; - } - } - - in = null; - } - } - - public synchronized void decRef() throws IOException { - assert refCount > 0 && (origNorm == null || origNorm.refCount > 0); - - if (--refCount == 0) { - if (origNorm != null) { - origNorm.decRef(); - origNorm = null; - } else { - closeInput(); - } - - if (bytes != null) { - assert bytesRef != null; - bytesRef.decrementAndGet(); - bytes = null; - bytesRef = null; - } else { - assert bytesRef == null; - } - } - } - - // Load & cache full bytes array. Returns bytes. - public synchronized byte[] bytes() throws IOException { - assert refCount > 0 && (origNorm == null || origNorm.refCount > 0); - if (bytes == null) { // value not yet read - assert bytesRef == null; - if (origNorm != null) { - // Ask origNorm to load so that for a series of - // reopened readers we share a single read-only - // byte[] - bytes = origNorm.bytes(); - bytesRef = origNorm.bytesRef; - bytesRef.incrementAndGet(); - - // Once we've loaded the bytes we no longer need - // origNorm: - origNorm.decRef(); - origNorm = null; - - } else { - // We are the origNorm, so load the bytes for real - // ourself: - final int count = owner.maxDoc(); - bytes = new byte[count]; - - // Since we are orig, in must not be null - assert in != null; - - // Read from disk. - synchronized(in) { - in.seek(normSeek); - in.readBytes(bytes, 0, count, false); - } - - bytesRef = new AtomicInteger(1); - closeInput(); - } - } - - return bytes; - } - - // Only for testing - AtomicInteger bytesRef() { - return bytesRef; - } - - // Called if we intend to change a norm value. We make a - // private copy of bytes if it's shared with others: - public synchronized byte[] copyOnWrite() throws IOException { - assert refCount > 0 && (origNorm == null || origNorm.refCount > 0); - bytes(); - assert bytes != null; - assert bytesRef != null; - if (bytesRef.get() > 1) { - // I cannot be the origNorm for another norm - // instance if I'm being changed. Ie, only the - // "head Norm" can be changed: - assert refCount == 1; - final AtomicInteger oldRef = bytesRef; - bytes = owner.cloneNormBytes(bytes); - bytesRef = new AtomicInteger(1); - oldRef.decrementAndGet(); - } - dirty = true; - return bytes; - } - - // Returns a copy of this Norm instance that shares - // IndexInput & bytes with the original one - @Override - public synchronized Object clone() { - assert refCount > 0 && (origNorm == null || origNorm.refCount > 0); - - SegmentNorms clone; - try { - clone = (SegmentNorms) super.clone(); - } catch (CloneNotSupportedException cnse) { - // Cannot happen - throw new RuntimeException("unexpected CloneNotSupportedException", cnse); - } - clone.refCount = 1; - - if (bytes != null) { - assert bytesRef != null; - assert origNorm == null; - - // Clone holds a reference to my bytes: - clone.bytesRef.incrementAndGet(); - } else { - assert bytesRef == null; - if (origNorm == null) { - // I become the origNorm for the clone: - clone.origNorm = this; - } - clone.origNorm.incRef(); - } - - // Only the origNorm will actually readBytes from in: - clone.in = null; - - return clone; - } - - // Flush all pending changes to the next generation - // separate norms file. - public void reWrite(SegmentInfo si) throws IOException { - assert refCount > 0 && (origNorm == null || origNorm.refCount > 0): "refCount=" + refCount + " origNorm=" + origNorm; - - // NOTE: norms are re-written in regular directory, not cfs - si.advanceNormGen(this.number); - final String normFileName = si.getNormFileName(this.number); - IndexOutput out = owner.directory().createOutput(normFileName, new IOContext(new FlushInfo(si.docCount, 0))); - boolean success = false; - try { - try { - out.writeBytes(SegmentNorms.NORMS_HEADER, 0, SegmentNorms.NORMS_HEADER.length); - out.writeBytes(bytes, owner.maxDoc()); - } finally { - out.close(); - } - success = true; - } finally { - if (!success) { - try { - owner.directory().deleteFile(normFileName); - } catch (Throwable t) { - // suppress this so we keep throwing the - // original exception - } - } - } - this.dirty = false; - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentReader.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentReader.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SegmentReader.java 2011-12-08 00:53:39.839793700 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SegmentReader.java 2011-12-08 19:31:49.842893300 +0100 @@ -20,15 +20,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IndexInput; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.codecs.StoredFieldsReader; import org.apache.lucene.index.codecs.PerDocValues; @@ -37,37 +34,30 @@ import org.apache.lucene.util.BitVector; import org.apache.lucene.util.Bits; import org.apache.lucene.util.CloseableThreadLocal; -import org.apache.lucene.util.StringHelper; /** * @lucene.experimental */ -public class SegmentReader extends IndexReader implements Cloneable { - protected boolean readOnly; +public final class SegmentReader extends IndexReader implements Cloneable { + private final boolean readOnly; private SegmentInfo si; private final ReaderContext readerContext = new AtomicReaderContext(this); - CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal(); - CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); + final CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal(); + final CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); volatile BitVector liveDocs; AtomicInteger liveDocsRef = null; + boolean hasChanges = false; private boolean liveDocsDirty = false; - private boolean normsDirty = false; - // TODO: we should move this tracking into SegmentInfo; - // this way SegmentInfo.toString shows pending deletes + // TODO: remove deletions from SR private int pendingDeleteCount; - private boolean rollbackHasChanges = false; private boolean rollbackDeletedDocsDirty = false; - private boolean rollbackNormsDirty = false; private SegmentInfo rollbackSegmentInfo; private int rollbackPendingDeleteCount; - - // optionally used for the .nrm file shared by multiple norms - IndexInput singleNormStream; - AtomicInteger singleNormRef; + // end TODO SegmentCoreReaders core; @@ -80,41 +70,39 @@ return core.getFieldsReaderOrig().clone(); } } - - Map norms = new HashMap(); /** * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException { - return get(readOnly, si.dir, si, true, termInfosIndexDivisor, context); + public static SegmentReader get(SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException { + return get(true, si, true, termInfosIndexDivisor, context); + } + + // TODO: remove deletions from SR + static SegmentReader getRW(SegmentInfo si, boolean doOpenStores, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException { + return get(false, si, doOpenStores, termInfosIndexDivisor, context); } /** * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static SegmentReader get(boolean readOnly, - Directory dir, + private static SegmentReader get(boolean readOnly, SegmentInfo si, boolean doOpenStores, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException { - SegmentReader instance = new SegmentReader(); - instance.readOnly = readOnly; - instance.si = si; + SegmentReader instance = new SegmentReader(readOnly, si); boolean success = false; - try { - instance.core = new SegmentCoreReaders(instance, dir, si, context, termInfosIndexDivisor); + instance.core = new SegmentCoreReaders(instance, si.dir, si, context, termInfosIndexDivisor); if (doOpenStores) { instance.core.openDocStores(si); } instance.loadLiveDocs(context); - instance.openNorms(instance.core.cfsDir, context); success = true; } finally { @@ -130,6 +118,11 @@ return instance; } + private SegmentReader(boolean readOnly, SegmentInfo si) { + this.readOnly = readOnly; + this.si = si; + } + void openDocStores() throws IOException { core.openDocStores(si); } @@ -158,7 +151,7 @@ private void loadLiveDocs(IOContext context) throws IOException { // NOTE: the bitvector is stored using the regular directory, not cfs - if (hasDeletions(si)) { + if (si.hasDeletions()) { liveDocs = new BitVector(directory(), si.getDelFileName(), new IOContext(context, true)); liveDocsRef = new AtomicInteger(1); assert checkLiveCounts(); @@ -168,27 +161,6 @@ } else assert si.getDelCount() == 0; } - - /** - * Clones the norm bytes. May be overridden by subclasses. New and experimental. - * @param bytes Byte array to clone - * @return New BitVector - */ - protected byte[] cloneNormBytes(byte[] bytes) { - byte[] cloneBytes = new byte[bytes.length]; - System.arraycopy(bytes, 0, cloneBytes, 0, bytes.length); - return cloneBytes; - } - - /** - * Clones the deleteDocs BitVector. May be overridden by subclasses. New and experimental. - * @param bv BitVector to clone - * @return New BitVector - */ - protected BitVector cloneDeletedDocs(BitVector bv) { - ensureOpen(); - return (BitVector)bv.clone(); - } @Override public final synchronized Object clone() { @@ -199,175 +171,16 @@ } } - @Override - public final synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException { - return reopenSegment(si, true, openReadOnly); + // used by DirectoryReader: + synchronized SegmentReader reopenSegment(SegmentInfo si, boolean doClone) throws CorruptIndexException, IOException { + return reopenSegment(si, doClone, true); } @Override - protected synchronized IndexReader doOpenIfChanged() - throws CorruptIndexException, IOException { + protected synchronized IndexReader doOpenIfChanged() throws CorruptIndexException, IOException { return reopenSegment(si, false, readOnly); } - - @Override - protected synchronized IndexReader doOpenIfChanged(boolean openReadOnly) - throws CorruptIndexException, IOException { - return reopenSegment(si, false, openReadOnly); - } - - synchronized SegmentReader reopenSegment(SegmentInfo si, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { - ensureOpen(); - boolean deletionsUpToDate = (this.si.hasDeletions() == si.hasDeletions()) - && (!si.hasDeletions() || this.si.getDelFileName().equals(si.getDelFileName())); - boolean normsUpToDate = true; - - Set fieldNormsChanged = new HashSet(); - for (FieldInfo fi : core.fieldInfos) { - int fieldNumber = fi.number; - if (!this.si.getNormFileName(fieldNumber).equals(si.getNormFileName(fieldNumber))) { - normsUpToDate = false; - fieldNormsChanged.add(fieldNumber); - } - } - - // if we're cloning we need to run through the reopenSegment logic - // also if both old and new readers aren't readonly, we clone to avoid sharing modifications - if (normsUpToDate && deletionsUpToDate && !doClone && openReadOnly && readOnly) { - return null; - } - - // When cloning, the incoming SegmentInfos should not - // have any changes in it: - assert !doClone || (normsUpToDate && deletionsUpToDate); - - // clone reader - SegmentReader clone = new SegmentReader(); - - boolean success = false; - try { - core.incRef(); - clone.core = core; - clone.readOnly = openReadOnly; - clone.si = si; - clone.pendingDeleteCount = pendingDeleteCount; - clone.readerFinishedListeners = readerFinishedListeners; - - if (!openReadOnly && hasChanges) { - // My pending changes transfer to the new reader - clone.liveDocsDirty = liveDocsDirty; - clone.normsDirty = normsDirty; - clone.hasChanges = hasChanges; - hasChanges = false; - } - - if (doClone) { - if (liveDocs != null) { - liveDocsRef.incrementAndGet(); - clone.liveDocs = liveDocs; - clone.liveDocsRef = liveDocsRef; - } - } else { - if (!deletionsUpToDate) { - // load deleted docs - assert clone.liveDocs == null; - clone.loadLiveDocs(IOContext.READ); - } else if (liveDocs != null) { - liveDocsRef.incrementAndGet(); - clone.liveDocs = liveDocs; - clone.liveDocsRef = liveDocsRef; - } - } - - clone.norms = new HashMap(); - - // Clone norms - for (FieldInfo fi : core.fieldInfos) { - // Clone unchanged norms to the cloned reader - if (doClone || !fieldNormsChanged.contains(fi.number)) { - final String curField = fi.name; - SegmentNorms norm = this.norms.get(curField); - if (norm != null) - clone.norms.put(curField, (SegmentNorms) norm.clone()); - } - } - - // If we are not cloning, then this will open anew - // any norms that have changed: - clone.openNorms(si.getUseCompoundFile() ? core.getCFSReader() : directory(), IOContext.DEFAULT); - - success = true; - } finally { - if (!success) { - // An exception occurred during reopen, we have to decRef the norms - // that we incRef'ed already and close singleNormsStream and FieldsReader - clone.decRef(); - } - } - - return clone; - } - - @Override - protected void doCommit(Map commitUserData) throws IOException { - if (hasChanges) { - startCommit(); - boolean success = false; - try { - commitChanges(commitUserData); - success = true; - } finally { - if (!success) { - rollbackCommit(); - } - } - } - } - - private synchronized void commitChanges(Map commitUserData) throws IOException { - if (liveDocsDirty) { // re-write deleted - si.advanceDelGen(); - - assert liveDocs.length() == si.docCount; - - // We can write directly to the actual name (vs to a - // .tmp & renaming it) because the file is not live - // until segments file is written: - final String delFileName = si.getDelFileName(); - boolean success = false; - try { - liveDocs.write(directory(), delFileName, IOContext.DEFAULT); - success = true; - } finally { - if (!success) { - try { - directory().deleteFile(delFileName); - } catch (Throwable t) { - // suppress this so we keep throwing the - // original exception - } - } - } - si.setDelCount(si.getDelCount()+pendingDeleteCount); - pendingDeleteCount = 0; - assert (maxDoc()-liveDocs.count()) == si.getDelCount(): "delete count mismatch during commit: info=" + si.getDelCount() + " vs BitVector=" + (maxDoc()-liveDocs.count()); - } else { - assert pendingDeleteCount == 0; - } - - if (normsDirty) { // re-write norms - si.initNormGen(); - for (final SegmentNorms norm : norms.values()) { - if (norm.dirty) { - norm.reWrite(si); - } - } - } - liveDocsDirty = false; - normsDirty = false; - hasChanges = false; - } - + /** @lucene.internal */ public StoredFieldsReader getFieldsReader() { return fieldsReaderLocal.get(); @@ -375,6 +188,10 @@ @Override protected void doClose() throws IOException { + if (hasChanges) { + doCommit(); + } + termVectorsLocal.close(); fieldsReaderLocal.close(); @@ -384,72 +201,17 @@ liveDocs = null; } - for (final SegmentNorms norm : norms.values()) { - norm.decRef(); - } if (core != null) { core.decRef(); } } - static boolean hasDeletions(SegmentInfo si) throws IOException { - // Don't call ensureOpen() here (it could affect performance) - return si.hasDeletions(); - } - @Override public boolean hasDeletions() { // Don't call ensureOpen() here (it could affect performance) return liveDocs != null; } - static boolean usesCompoundFile(SegmentInfo si) throws IOException { - return si.getUseCompoundFile(); - } - - static boolean hasSeparateNorms(SegmentInfo si) throws IOException { - return si.hasSeparateNorms(); - } - - @Override - protected void doDelete(int docNum) { - if (liveDocs == null) { - liveDocs = new BitVector(maxDoc()); - liveDocs.setAll(); - liveDocsRef = new AtomicInteger(1); - } - // there is more than 1 SegmentReader with a reference to this - // liveDocs BitVector so decRef the current liveDocsRef, - // clone the BitVector, create a new liveDocsRef - if (liveDocsRef.get() > 1) { - AtomicInteger oldRef = liveDocsRef; - liveDocs = cloneDeletedDocs(liveDocs); - liveDocsRef = new AtomicInteger(1); - oldRef.decrementAndGet(); - } - liveDocsDirty = true; - if (liveDocs.getAndClear(docNum)) { - pendingDeleteCount++; - } - } - - @Override - protected void doUndeleteAll() { - liveDocsDirty = false; - if (liveDocs != null) { - assert liveDocsRef != null; - liveDocsRef.decrementAndGet(); - liveDocs = null; - liveDocsRef = null; - pendingDeleteCount = 0; - si.clearDelGen(); - si.setDelCount(0); - } else { - assert liveDocsRef == null; - assert pendingDeleteCount == 0; - } - } - List files() throws IOException { return new ArrayList(si.files()); } @@ -547,107 +309,14 @@ @Override public boolean hasNorms(String field) { ensureOpen(); - return norms.containsKey(field); + FieldInfo fi = core.fieldInfos.fieldInfo(field); + return fi != null && fi.isIndexed && !fi.omitNorms; } @Override public byte[] norms(String field) throws IOException { ensureOpen(); - final SegmentNorms norm = norms.get(field); - if (norm == null) { - // not indexed, or norms not stored - return null; - } - return norm.bytes(); - } - - @Override - protected void doSetNorm(int doc, String field, byte value) - throws IOException { - SegmentNorms norm = norms.get(field); - if (norm == null) { - // field does not store norms - throw new IllegalStateException("Cannot setNorm for field " + field + ": norms were omitted"); - } - - normsDirty = true; - norm.copyOnWrite()[doc] = value; // set the value - } - - private void openNorms(Directory cfsDir, IOContext context) throws IOException { - boolean normsInitiallyEmpty = norms.isEmpty(); // only used for assert - long nextNormSeek = SegmentNorms.NORMS_HEADER.length; //skip header (header unused for now) - int maxDoc = maxDoc(); - for (FieldInfo fi : core.fieldInfos) { - if (norms.containsKey(fi.name)) { - // in case this SegmentReader is being re-opened, we might be able to - // reuse some norm instances and skip loading them here - continue; - } - if (fi.isIndexed && !fi.omitNorms) { - Directory d = directory(); - String fileName = si.getNormFileName(fi.number); - if (!si.hasSeparateNorms(fi.number)) { - d = cfsDir; - } - - // singleNormFile means multiple norms share this file - boolean singleNormFile = IndexFileNames.matchesExtension(fileName, IndexFileNames.NORMS_EXTENSION); - IndexInput normInput = null; - long normSeek; - - if (singleNormFile) { - normSeek = nextNormSeek; - if (singleNormStream == null) { - singleNormStream = d.openInput(fileName, context); - singleNormRef = new AtomicInteger(1); - } else { - singleNormRef.incrementAndGet(); - } - // All norms in the .nrm file can share a single IndexInput since - // they are only used in a synchronized context. - // If this were to change in the future, a clone could be done here. - normInput = singleNormStream; - } else { - normInput = d.openInput(fileName, context); - // if the segment was created in 3.2 or after, we wrote the header for sure, - // and don't need to do the sketchy file size check. otherwise, we check - // if the size is exactly equal to maxDoc to detect a headerless file. - // NOTE: remove this check in Lucene 5.0! - String version = si.getVersion(); - final boolean isUnversioned = - (version == null || StringHelper.getVersionComparator().compare(version, "3.2") < 0) - && normInput.length() == maxDoc(); - if (isUnversioned) { - normSeek = 0; - } else { - normSeek = SegmentNorms.NORMS_HEADER.length; - } - } - - norms.put(fi.name, new SegmentNorms(normInput, fi.number, normSeek, this)); - nextNormSeek += maxDoc; // increment also if some norms are separate - } - } - assert singleNormStream == null || !normsInitiallyEmpty || nextNormSeek == singleNormStream.length(); - } - - // for testing only - boolean normsClosed() { - if (singleNormStream != null) { - return false; - } - for (final SegmentNorms norm : norms.values()) { - if (norm.refCount > 0) { - return false; - } - } - return true; - } - - // for testing only - boolean normsClosed(String field) { - return norms.get(field).refCount == 0; + return core.norms.norms(field); } /** @@ -689,7 +358,6 @@ return termVectorsReader.get(docID); } - /** {@inheritDoc} */ @Override public String toString() { final StringBuilder buffer = new StringBuilder(); @@ -724,28 +392,6 @@ si = info; } - void startCommit() { - rollbackSegmentInfo = (SegmentInfo) si.clone(); - rollbackHasChanges = hasChanges; - rollbackDeletedDocsDirty = liveDocsDirty; - rollbackNormsDirty = normsDirty; - rollbackPendingDeleteCount = pendingDeleteCount; - for (SegmentNorms norm : norms.values()) { - norm.rollbackDirty = norm.dirty; - } - } - - void rollbackCommit() { - si.reset(rollbackSegmentInfo); - hasChanges = rollbackHasChanges; - liveDocsDirty = rollbackDeletedDocsDirty; - normsDirty = rollbackNormsDirty; - pendingDeleteCount = rollbackPendingDeleteCount; - for (SegmentNorms norm : norms.values()) { - norm.dirty = norm.rollbackDirty; - } - } - /** Returns the directory this index resides in. */ @Override public Directory directory() { @@ -783,4 +429,178 @@ ensureOpen(); return core.perDocProducer; } + + /** + * Clones the deleteDocs BitVector. May be overridden by subclasses. New and experimental. + * @param bv BitVector to clone + * @return New BitVector + */ + // TODO: remove deletions from SR + BitVector cloneDeletedDocs(BitVector bv) { + ensureOpen(); + return (BitVector)bv.clone(); + } + + // TODO: remove deletions from SR + final synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException { + return reopenSegment(si, true, openReadOnly); + } + + // TODO: remove deletions from SR + private synchronized SegmentReader reopenSegment(SegmentInfo si, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { + ensureOpen(); + boolean deletionsUpToDate = (this.si.hasDeletions() == si.hasDeletions()) + && (!si.hasDeletions() || this.si.getDelFileName().equals(si.getDelFileName())); + + // if we're cloning we need to run through the reopenSegment logic + // also if both old and new readers aren't readonly, we clone to avoid sharing modifications + if (deletionsUpToDate && !doClone && openReadOnly && readOnly) { + return null; + } + + // When cloning, the incoming SegmentInfos should not + // have any changes in it: + assert !doClone || (deletionsUpToDate); + + // clone reader + SegmentReader clone = new SegmentReader(openReadOnly, si); + + boolean success = false; + try { + core.incRef(); + clone.core = core; + clone.pendingDeleteCount = pendingDeleteCount; + clone.readerFinishedListeners = readerFinishedListeners; + + if (!openReadOnly && hasChanges) { + // My pending changes transfer to the new reader + clone.liveDocsDirty = liveDocsDirty; + clone.hasChanges = hasChanges; + hasChanges = false; + } + + if (doClone) { + if (liveDocs != null) { + liveDocsRef.incrementAndGet(); + clone.liveDocs = liveDocs; + clone.liveDocsRef = liveDocsRef; + } + } else { + if (!deletionsUpToDate) { + // load deleted docs + assert clone.liveDocs == null; + clone.loadLiveDocs(IOContext.READ); + } else if (liveDocs != null) { + liveDocsRef.incrementAndGet(); + clone.liveDocs = liveDocs; + clone.liveDocsRef = liveDocsRef; + } + } + success = true; + } finally { + if (!success) { + // An exception occurred during reopen, we have to decRef the norms + // that we incRef'ed already and close singleNormsStream and FieldsReader + clone.decRef(); + } + } + + return clone; + } + + // TODO: remove deletions from SR + void doCommit() throws IOException { + assert hasChanges; + startCommit(); + boolean success = false; + try { + commitChanges(); + success = true; + } finally { + if (!success) { + rollbackCommit(); + } + } + } + + // TODO: remove deletions from SR + private void startCommit() { + rollbackSegmentInfo = (SegmentInfo) si.clone(); + rollbackHasChanges = hasChanges; + rollbackDeletedDocsDirty = liveDocsDirty; + rollbackPendingDeleteCount = pendingDeleteCount; + } + + // TODO: remove deletions from SR + private void rollbackCommit() { + si.reset(rollbackSegmentInfo); + hasChanges = rollbackHasChanges; + liveDocsDirty = rollbackDeletedDocsDirty; + pendingDeleteCount = rollbackPendingDeleteCount; + } + + // TODO: remove deletions from SR + private synchronized void commitChanges() throws IOException { + if (liveDocsDirty) { // re-write deleted + si.advanceDelGen(); + + assert liveDocs.length() == si.docCount; + + // We can write directly to the actual name (vs to a + // .tmp & renaming it) because the file is not live + // until segments file is written: + final String delFileName = si.getDelFileName(); + boolean success = false; + try { + liveDocs.write(directory(), delFileName, IOContext.DEFAULT); + success = true; + } finally { + if (!success) { + try { + directory().deleteFile(delFileName); + } catch (Throwable t) { + // suppress this so we keep throwing the + // original exception + } + } + } + si.setDelCount(si.getDelCount()+pendingDeleteCount); + pendingDeleteCount = 0; + assert (maxDoc()-liveDocs.count()) == si.getDelCount(): "delete count mismatch during commit: info=" + si.getDelCount() + " vs BitVector=" + (maxDoc()-liveDocs.count()); + } else { + assert pendingDeleteCount == 0; + } + + liveDocsDirty = false; + hasChanges = false; + } + + // TODO: remove deletions from SR + synchronized void deleteDocument(int docNum) throws IOException { + ensureOpen(); + hasChanges = true; + doDelete(docNum); + } + + // TODO: remove deletions from SR + void doDelete(int docNum) { + if (liveDocs == null) { + liveDocs = new BitVector(maxDoc()); + liveDocs.setAll(); + liveDocsRef = new AtomicInteger(1); + } + // there is more than 1 SegmentReader with a reference to this + // liveDocs BitVector so decRef the current liveDocsRef, + // clone the BitVector, create a new liveDocsRef + if (liveDocsRef.get() > 1) { + AtomicInteger oldRef = liveDocsRef; + liveDocs = cloneDeletedDocs(liveDocs); + liveDocsRef = new AtomicInteger(1); + oldRef.decrementAndGet(); + } + liveDocsDirty = true; + if (liveDocs.getAndClear(docNum)) { + pendingDeleteCount++; + } + } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java 2011-10-21 19:50:46.265312800 +0200 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java 2011-12-08 19:31:55.889239100 +0100 @@ -109,13 +109,4 @@ ensureOpen(); return readerContext; } - - @Override - protected void doSetNorm(int n, String field, byte value) - throws CorruptIndexException, IOException { - synchronized(normsCache) { - normsCache.remove(field); - } - in.doSetNorm(n, field, value); - } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/StaleReaderException.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/StaleReaderException.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/StaleReaderException.java 2011-10-19 14:22:11.551246200 +0200 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/StaleReaderException.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.index; - -import java.io.IOException; - -/** - * This exception is thrown when an {@link IndexReader} - * tries to make changes to the index (via {@link - * IndexReader#deleteDocument}, {@link - * IndexReader#undeleteAll} or {@link IndexReader#setNorm}) - * but changes have already been committed to the index - * since this reader was instantiated. When this happens - * you must open a new reader on the current index to make - * the changes. - */ -public class StaleReaderException extends IOException { - public StaleReaderException(String message) { - super(message); - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/Codec.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/Codec.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/Codec.java 2011-11-16 23:30:36.680125300 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/Codec.java 2011-12-08 19:31:54.939184700 +0100 @@ -51,6 +51,7 @@ // TODO: segmentInfosFormat should be allowed to declare additional files // if it wants, in addition to segments_N docValuesFormat().files(dir, info, files); + normsFormat().files(dir, info, files); } /** Encodes/decodes postings */ @@ -71,6 +72,9 @@ /** Encodes/decodes segments file */ public abstract SegmentInfosFormat segmentInfosFormat(); + /** Encodes/decodes document normalization values */ + public abstract NormsFormat normsFormat(); + /** looks up a codec by name */ public static Codec forName(String name) { return loader.lookup(name); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/NormsFormat.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/NormsFormat.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/NormsFormat.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/NormsFormat.java 2011-12-08 19:31:54.439156100 +0100 @@ -0,0 +1,44 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +/** + * format for normalization factors + */ +public abstract class NormsFormat { + /** Note: separateNormsDir should not be used! */ + public abstract NormsReader normsReader(Directory dir, SegmentInfo info, FieldInfos fields, IOContext context, Directory separateNormsDir) throws IOException; + public abstract NormsWriter normsWriter(SegmentWriteState state) throws IOException; + public abstract void files(Directory dir, SegmentInfo info, Set files) throws IOException; + + /** + * Note: this should not be overridden! + * @deprecated + */ + @Deprecated + public void separateFiles(Directory dir, SegmentInfo info, Set files) throws IOException {}; +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/NormsReader.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/NormsReader.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/NormsReader.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/NormsReader.java 2011-12-08 19:31:55.024189600 +0100 @@ -0,0 +1,26 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Closeable; +import java.io.IOException; + +//simple api just for now before switching to docvalues apis +public abstract class NormsReader implements Closeable { + public abstract byte[] norms(String name) throws IOException; +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/NormsWriter.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/NormsWriter.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/NormsWriter.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/NormsWriter.java 2011-12-08 19:31:55.423212400 +0100 @@ -0,0 +1,70 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.util.Bits; + +// simple api just for now before switching to docvalues apis +public abstract class NormsWriter implements Closeable { + + // TODO: I think IW should set info.normValueType from Similarity, + // and then this method just returns DocValuesConsumer + public abstract void startField(FieldInfo info) throws IOException; + public abstract void writeNorm(byte norm) throws IOException; + public abstract void finish(int numDocs) throws IOException; + + public int merge(MergeState mergeState) throws IOException { + int numMergedDocs = 0; + for (FieldInfo fi : mergeState.fieldInfos) { + if (fi.isIndexed && !fi.omitNorms) { + startField(fi); + int numMergedDocsForField = 0; + for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) { + final int maxDoc = reader.reader.maxDoc(); + byte normBuffer[] = reader.reader.norms(fi.name); + if (normBuffer == null) { + // Can be null if this segment doesn't have + // any docs with this field + normBuffer = new byte[maxDoc]; + Arrays.fill(normBuffer, (byte)0); + } + // this segment has deleted docs, so we have to + // check for every doc if it is deleted or not + final Bits liveDocs = reader.liveDocs; + for (int k = 0; k < maxDoc; k++) { + if (liveDocs == null || liveDocs.get(k)) { + writeNorm(normBuffer[k]); + numMergedDocsForField++; + } + } + mergeState.checkAbort.work(maxDoc); + } + assert numMergedDocs == 0 || numMergedDocs == numMergedDocsForField; + numMergedDocs = numMergedDocsForField; + } + } + finish(numMergedDocs); + return numMergedDocs; + } +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java 2011-12-07 00:03:06.358815300 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java 2011-12-08 19:31:53.368094900 +0100 @@ -20,6 +20,7 @@ import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.codecs.FieldInfosFormat; +import org.apache.lucene.index.codecs.NormsFormat; import org.apache.lucene.index.codecs.StoredFieldsFormat; import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.SegmentInfosFormat; @@ -27,6 +28,7 @@ import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; import org.apache.lucene.index.codecs.lucene40.Lucene40FieldInfosFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40DocValuesFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40NormsFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40StoredFieldsFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40TermVectorsFormat; @@ -47,6 +49,7 @@ private final FieldInfosFormat fieldInfos = new Lucene40FieldInfosFormat(); private final TermVectorsFormat vectors = new Lucene40TermVectorsFormat(); private final DocValuesFormat docValues = new Lucene40DocValuesFormat(); + private final NormsFormat norms = new Lucene40NormsFormat(); @Override public PostingsFormat postingsFormat() { @@ -77,4 +80,9 @@ public FieldInfosFormat fieldInfosFormat() { return fieldInfos; } + + @Override + public NormsFormat normsFormat() { + return norms; + } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java 2011-12-03 13:28:30.755321900 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java 2011-12-08 19:31:54.211143100 +0100 @@ -26,6 +26,7 @@ import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.codecs.FieldInfosFormat; +import org.apache.lucene.index.codecs.NormsFormat; import org.apache.lucene.index.codecs.StoredFieldsFormat; import org.apache.lucene.index.codecs.PerDocConsumer; import org.apache.lucene.index.codecs.PerDocValues; @@ -33,6 +34,7 @@ import org.apache.lucene.index.codecs.SegmentInfosFormat; import org.apache.lucene.index.codecs.TermVectorsFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40FieldInfosFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40NormsFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40SegmentInfosFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40StoredFieldsFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40TermVectorsFormat; @@ -62,6 +64,9 @@ // this way IR.commit fails on delete/undelete/setNorm/etc ? private final SegmentInfosFormat infosFormat = new Lucene40SegmentInfosFormat(); + // TODO: this should really be a different impl + private final NormsFormat normsFormat = new Lucene40NormsFormat(); + // 3.x doesn't support docvalues private final DocValuesFormat docValuesFormat = new DocValuesFormat() { @Override @@ -107,4 +112,9 @@ public SegmentInfosFormat segmentInfosFormat() { return infosFormat; } + + @Override + public NormsFormat normsFormat() { + return normsFormat; + } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java 2011-12-03 13:28:29.977277400 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java 2011-12-08 19:31:51.598993700 +0100 @@ -20,6 +20,7 @@ import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.codecs.FieldInfosFormat; +import org.apache.lucene.index.codecs.NormsFormat; import org.apache.lucene.index.codecs.StoredFieldsFormat; import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.SegmentInfosFormat; @@ -40,6 +41,7 @@ private final FieldInfosFormat fieldInfosFormat = new Lucene40FieldInfosFormat(); private final DocValuesFormat docValuesFormat = new Lucene40DocValuesFormat(); private final SegmentInfosFormat infosFormat = new Lucene40SegmentInfosFormat(); + private final NormsFormat normsFormat = new Lucene40NormsFormat(); private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { @Override public PostingsFormat getPostingsFormatForField(String field) { @@ -81,6 +83,11 @@ return infosFormat; } + @Override + public NormsFormat normsFormat() { + return normsFormat; + } + /** Returns the postings format that should be used for writing * new segments of field. * diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsFormat.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsFormat.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsFormat.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsFormat.java 2011-12-08 19:31:51.456985600 +0100 @@ -0,0 +1,53 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.NormsFormat; +import org.apache.lucene.index.codecs.NormsReader; +import org.apache.lucene.index.codecs.NormsWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +public class Lucene40NormsFormat extends NormsFormat { + + @Override + public NormsReader normsReader(Directory dir, SegmentInfo info, FieldInfos fields, IOContext context, Directory separateNormsDir) throws IOException { + return new Lucene40NormsReader(dir, info, fields, context, separateNormsDir); + } + + @Override + public NormsWriter normsWriter(SegmentWriteState state) throws IOException { + return new Lucene40NormsWriter(state.directory, state.segmentName, state.context); + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + Lucene40NormsReader.files(dir, info, files); + } + + @Override + public void separateFiles(Directory dir, SegmentInfo info, Set files) throws IOException { + Lucene40NormsReader.separateFiles(dir, info, files); + } +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsReader.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsReader.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsReader.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsReader.java 2011-12-08 19:31:51.698999400 +0100 @@ -0,0 +1,196 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.codecs.NormsReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.MapBackedSet; +import org.apache.lucene.util.StringHelper; + +public class Lucene40NormsReader extends NormsReader { + // this would be replaced by Source/SourceCache in a dv impl. + // for now we have our own mini-version + final Map norms = new HashMap(); + // any .nrm or .sNN files we have open at any time. + // TODO: just a list, and double-close() separate norms files? + final Set openFiles = new MapBackedSet(new IdentityHashMap()); + // points to a singleNormFile + IndexInput singleNormStream; + final int maxdoc; + + // note: just like segmentreader in 3.x, we open up all the files here (including separate norms) up front. + // but we just don't do any seeks or reading yet. + public Lucene40NormsReader(Directory dir, SegmentInfo info, FieldInfos fields, IOContext context, Directory separateNormsDir) throws IOException { + maxdoc = info.docCount; + String segmentName = info.name; + Map normGen = info.getNormGen(); + boolean success = false; + try { + long nextNormSeek = Lucene40NormsWriter.NORMS_HEADER.length; //skip header (header unused for now) + for (FieldInfo fi : fields) { + if (fi.isIndexed && !fi.omitNorms) { + String fileName = getNormFilename(segmentName, normGen, fi.number); + Directory d = hasSeparateNorms(normGen, fi.number) ? separateNormsDir : dir; + + // singleNormFile means multiple norms share this file + boolean singleNormFile = IndexFileNames.matchesExtension(fileName, Lucene40NormsWriter.NORMS_EXTENSION); + IndexInput normInput = null; + long normSeek; + + if (singleNormFile) { + normSeek = nextNormSeek; + if (singleNormStream == null) { + singleNormStream = d.openInput(fileName, context); + openFiles.add(singleNormStream); + } + // All norms in the .nrm file can share a single IndexInput since + // they are only used in a synchronized context. + // If this were to change in the future, a clone could be done here. + normInput = singleNormStream; + } else { + normInput = d.openInput(fileName, context); + openFiles.add(normInput); + // if the segment was created in 3.2 or after, we wrote the header for sure, + // and don't need to do the sketchy file size check. otherwise, we check + // if the size is exactly equal to maxDoc to detect a headerless file. + // NOTE: remove this check in Lucene 5.0! + String version = info.getVersion(); + final boolean isUnversioned = + (version == null || StringHelper.getVersionComparator().compare(version, "3.2") < 0) + && normInput.length() == maxdoc; + if (isUnversioned) { + normSeek = 0; + } else { + normSeek = Lucene40NormsWriter.NORMS_HEADER.length; + } + } + + Norm norm = new Norm(); + norm.file = normInput; + norm.offset = normSeek; + norms.put(fi.name, norm); + nextNormSeek += maxdoc; // increment also if some norms are separate + } + } + // TODO: change to a real check? see LUCENE-3619 + assert singleNormStream == null || nextNormSeek == singleNormStream.length(); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(openFiles); + } + } + } + + @Override + public byte[] norms(String name) throws IOException { + Norm norm = norms.get(name); + return norm == null ? null : norm.bytes(); + } + + + @Override + public void close() throws IOException { + try { + IOUtils.close(openFiles); + } finally { + norms.clear(); + openFiles.clear(); + } + } + + private static String getNormFilename(String segmentName, Map normGen, int number) { + if (hasSeparateNorms(normGen, number)) { + return IndexFileNames.fileNameFromGeneration(segmentName, Lucene40NormsWriter.SEPARATE_NORMS_EXTENSION + number, normGen.get(number)); + } else { + // single file for all norms + return IndexFileNames.fileNameFromGeneration(segmentName, Lucene40NormsWriter.NORMS_EXTENSION, SegmentInfo.WITHOUT_GEN); + } + } + + private static boolean hasSeparateNorms(Map normGen, int number) { + if (normGen == null) { + return false; + } + + Long gen = normGen.get(number); + return gen != null && gen.longValue() != SegmentInfo.NO; + } + + class Norm { + IndexInput file; + long offset; + byte bytes[]; + + synchronized byte[] bytes() throws IOException { + if (bytes == null) { + bytes = new byte[maxdoc]; + // some norms share fds + synchronized(file) { + file.seek(offset); + file.readBytes(bytes, 0, bytes.length, false); + } + // we are done with this file + if (file != singleNormStream) { + openFiles.remove(file); + file.close(); + file = null; + } + } + return bytes; + } + } + + static void files(Directory dir, SegmentInfo info, Set files) throws IOException { + // TODO: This is what SI always did... but we can do this cleaner? + // like first FI that has norms but doesn't have separate norms? + final String normsFileName = IndexFileNames.segmentFileName(info.name, "", Lucene40NormsWriter.NORMS_EXTENSION); + if (dir.fileExists(normsFileName)) { + files.add(normsFileName); + } + } + + /** @deprecated */ + @Deprecated + static void separateFiles(Directory dir, SegmentInfo info, Set files) throws IOException { + Map normGen = info.getNormGen(); + if (normGen != null) { + for (Entry entry : normGen.entrySet()) { + long gen = entry.getValue(); + if (gen >= SegmentInfo.YES) { + // Definitely a separate norm file, with generation: + files.add(IndexFileNames.fileNameFromGeneration(info.name, Lucene40NormsWriter.SEPARATE_NORMS_EXTENSION + entry.getKey(), gen)); + } + } + } + } +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsWriter.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsWriter.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsWriter.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsWriter.java 2011-12-08 19:31:51.938013100 +0100 @@ -0,0 +1,130 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.codecs.NormsWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.IOUtils; + +public class Lucene40NormsWriter extends NormsWriter { + private IndexOutput out; + private int normCount = 0; + + /** norms header placeholder */ + static final byte[] NORMS_HEADER = new byte[]{'N','R','M',-1}; + + /** Extension of norms file */ + static final String NORMS_EXTENSION = "nrm"; + + /** Extension of separate norms file + * @deprecated */ + @Deprecated + static final String SEPARATE_NORMS_EXTENSION = "s"; + + public Lucene40NormsWriter(Directory directory, String segment, IOContext context) throws IOException { + final String normsFileName = IndexFileNames.segmentFileName(segment, "", NORMS_EXTENSION); + boolean success = false; + try { + out = directory.createOutput(normsFileName, context); + out.writeBytes(NORMS_HEADER, 0, NORMS_HEADER.length); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } + } + } + + @Override + public void startField(FieldInfo info) throws IOException { + assert info.omitNorms == false; + normCount++; + } + + @Override + public void writeNorm(byte norm) throws IOException { + out.writeByte(norm); + } + + @Override + public void finish(int numDocs) throws IOException { + if (4+normCount*(long)numDocs != out.getFilePointer()) { + throw new RuntimeException(".nrm file size mismatch: expected=" + (4+normCount*(long)numDocs) + " actual=" + out.getFilePointer()); + } + } + + /** we override merge and bulk-merge norms when there are no deletions */ + @Override + public int merge(MergeState mergeState) throws IOException { + int numMergedDocs = 0; + for (FieldInfo fi : mergeState.fieldInfos) { + if (fi.isIndexed && !fi.omitNorms) { + startField(fi); + int numMergedDocsForField = 0; + for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) { + final int maxDoc = reader.reader.maxDoc(); + byte normBuffer[] = reader.reader.norms(fi.name); + if (normBuffer == null) { + // Can be null if this segment doesn't have + // any docs with this field + normBuffer = new byte[maxDoc]; + Arrays.fill(normBuffer, (byte)0); + } + if (reader.liveDocs == null) { + //optimized case for segments without deleted docs + out.writeBytes(normBuffer, maxDoc); + numMergedDocsForField += maxDoc; + } else { + // this segment has deleted docs, so we have to + // check for every doc if it is deleted or not + final Bits liveDocs = reader.liveDocs; + for (int k = 0; k < maxDoc; k++) { + if (liveDocs.get(k)) { + numMergedDocsForField++; + out.writeByte(normBuffer[k]); + } + } + } + mergeState.checkAbort.work(maxDoc); + } + assert numMergedDocs == 0 || numMergedDocs == numMergedDocsForField; + numMergedDocs = numMergedDocsForField; + } + } + finish(numMergedDocs); + return numMergedDocs; + } + + @Override + public void close() throws IOException { + try { + IOUtils.close(out); + } finally { + out = null; + } + } +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java 2011-12-03 13:28:30.335297900 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java 2011-12-08 19:31:52.879066900 +0100 @@ -20,6 +20,7 @@ import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.codecs.FieldInfosFormat; +import org.apache.lucene.index.codecs.NormsFormat; import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.SegmentInfosFormat; import org.apache.lucene.index.codecs.StoredFieldsFormat; @@ -40,6 +41,8 @@ private final TermVectorsFormat vectorsFormat = new SimpleTextTermVectorsFormat(); // TODO: need a plain-text impl private final DocValuesFormat docValues = new Lucene40DocValuesFormat(); + // TODO: need a plain-text impl (using the above) + private final NormsFormat normsFormat = new SimpleTextNormsFormat(); public SimpleTextCodec() { super("SimpleText"); @@ -74,4 +77,9 @@ public SegmentInfosFormat segmentInfosFormat() { return segmentInfos; } + + @Override + public NormsFormat normsFormat() { + return normsFormat; + } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsFormat.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsFormat.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsFormat.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsFormat.java 2011-12-08 19:31:52.720057800 +0100 @@ -0,0 +1,54 @@ +package org.apache.lucene.index.codecs.simpletext; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.NormsFormat; +import org.apache.lucene.index.codecs.NormsReader; +import org.apache.lucene.index.codecs.NormsWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +/** + * plain-text norms format + *

+ * FOR RECREATIONAL USE ONLY + * @lucene.experimental + */ +public class SimpleTextNormsFormat extends NormsFormat { + + @Override + public NormsReader normsReader(Directory dir, SegmentInfo info, FieldInfos fields, IOContext context, Directory separateNormsDir) throws IOException { + return new SimpleTextNormsReader(dir, info, fields, context); + } + + @Override + public NormsWriter normsWriter(SegmentWriteState state) throws IOException { + return new SimpleTextNormsWriter(state.directory, state.segmentName, state.context); + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + SimpleTextNormsReader.files(dir, info, files); + } +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsReader.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsReader.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsReader.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsReader.java 2011-12-08 19:31:52.253031100 +0100 @@ -0,0 +1,106 @@ +package org.apache.lucene.index.codecs.simpletext; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.codecs.NormsReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.StringHelper; + +import static org.apache.lucene.index.codecs.simpletext.SimpleTextNormsWriter.*; + +/** + * Reads plain-text norms + *

+ * FOR RECREATIONAL USE ONLY + * @lucene.experimental + */ +public class SimpleTextNormsReader extends NormsReader { + private Map norms = new HashMap(); + + public SimpleTextNormsReader(Directory directory, SegmentInfo si, FieldInfos fields, IOContext context) throws IOException { + if (fields.hasNorms()) { + readNorms(directory.openInput(IndexFileNames.segmentFileName(si.name, "", NORMS_EXTENSION), context), si.docCount); + } + } + + // we read in all the norms up front into a hashmap + private void readNorms(IndexInput in, int maxDoc) throws IOException { + BytesRef scratch = new BytesRef(); + boolean success = false; + try { + SimpleTextUtil.readLine(in, scratch); + while (!scratch.equals(END)) { + assert StringHelper.startsWith(scratch, FIELD); + String fieldName = readString(FIELD.length, scratch); + byte bytes[] = new byte[maxDoc]; + for (int i = 0; i < bytes.length; i++) { + SimpleTextUtil.readLine(in, scratch); + assert StringHelper.startsWith(scratch, DOC); + SimpleTextUtil.readLine(in, scratch); + assert StringHelper.startsWith(scratch, NORM); + bytes[i] = scratch.bytes[scratch.offset + NORM.length]; + } + norms.put(fieldName, bytes); + SimpleTextUtil.readLine(in, scratch); + assert StringHelper.startsWith(scratch, FIELD) || scratch.equals(END); + } + success = true; + } finally { + if (success) { + IOUtils.close(in); + } else { + IOUtils.closeWhileHandlingException(in); + } + } + } + + @Override + public byte[] norms(String name) throws IOException { + return norms.get(name); + } + + @Override + public void close() throws IOException { + norms = null; + } + + static void files(Directory dir, SegmentInfo info, Set files) throws IOException { + // TODO: This is what SI always did... but we can do this cleaner? + // like first FI that has norms but doesn't have separate norms? + final String normsFileName = IndexFileNames.segmentFileName(info.name, "", SimpleTextNormsWriter.NORMS_EXTENSION); + if (dir.fileExists(normsFileName)) { + files.add(normsFileName); + } + } + + private String readString(int offset, BytesRef scratch) { + return new String(scratch.bytes, scratch.offset+offset, scratch.length-offset, IOUtils.CHARSET_UTF_8); + } +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsWriter.java trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsWriter.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsWriter.java 1970-01-01 01:00:00.000000000 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextNormsWriter.java 2011-12-08 19:31:52.508045700 +0100 @@ -0,0 +1,114 @@ +package org.apache.lucene.index.codecs.simpletext; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.codecs.NormsWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; + +/** + * Writes plain-text norms + *

+ * FOR RECREATIONAL USE ONLY + * @lucene.experimental + */ +public class SimpleTextNormsWriter extends NormsWriter { + private IndexOutput out; + private int docid = 0; + + /** Extension of norms file */ + static final String NORMS_EXTENSION = "len"; + + private final BytesRef scratch = new BytesRef(); + + final static BytesRef END = new BytesRef("END"); + final static BytesRef FIELD = new BytesRef("field "); + final static BytesRef DOC = new BytesRef(" doc "); + final static BytesRef NORM = new BytesRef(" norm "); + + public SimpleTextNormsWriter(Directory directory, String segment, IOContext context) throws IOException { + final String normsFileName = IndexFileNames.segmentFileName(segment, "", NORMS_EXTENSION); + out = directory.createOutput(normsFileName, context); + } + + @Override + public void startField(FieldInfo info) throws IOException { + assert info.omitNorms == false; + docid = 0; + write(FIELD); + write(info.name); + newLine(); + } + + @Override + public void writeNorm(byte norm) throws IOException { + write(DOC); + write(Integer.toString(docid)); + newLine(); + + write(NORM); + write(norm); + newLine(); + docid++; + } + + @Override + public void finish(int numDocs) throws IOException { + if (docid != numDocs) { + throw new RuntimeException("mergeNorms produced an invalid result: docCount is " + numDocs + + " but only saw " + docid + " file=" + out.toString() + "; now aborting this merge to prevent index corruption"); + } + write(END); + newLine(); + } + + @Override + public void close() throws IOException { + try { + IOUtils.close(out); + } finally { + out = null; + } + } + + private void write(String s) throws IOException { + SimpleTextUtil.write(out, s, scratch); + } + + private void write(BytesRef bytes) throws IOException { + SimpleTextUtil.write(out, bytes); + } + + private void write(byte b) throws IOException { + scratch.grow(1); + scratch.bytes[scratch.offset] = b; + scratch.length = 1; + SimpleTextUtil.write(out, scratch); + } + + private void newLine() throws IOException { + SimpleTextUtil.writeNewline(out); + } +} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/java/org/apache/lucene/search/SearcherManager.java trunk-lusolr3/lucene/src/java/org/apache/lucene/search/SearcherManager.java --- trunk-lusolr4/lucene/src/java/org/apache/lucene/search/SearcherManager.java 2011-12-01 13:37:57.225558300 +0100 +++ trunk-lusolr3/lucene/src/java/org/apache/lucene/search/SearcherManager.java 2011-12-08 19:31:45.215628600 +0100 @@ -128,7 +128,7 @@ ExecutorService es) throws IOException { this.es = es; this.warmer = warmer; - currentSearcher = new IndexSearcher(IndexReader.open(dir, true), es); + currentSearcher = new IndexSearcher(IndexReader.open(dir), es); } /** diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java trunk-lusolr3/lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java 2011-10-21 19:50:40.822001400 +0200 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/document/TestBinaryDocument.java 2011-12-08 19:31:44.342578700 +0100 @@ -71,13 +71,6 @@ writer.close(); reader.close(); - - reader = IndexReader.open(dir, false); - /** delete the document from index */ - reader.deleteDocument(0); - assertEquals(0, reader.numDocs()); - - reader.close(); dir.close(); } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java 2011-12-08 00:53:39.741788100 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java 2011-12-08 19:31:35.529074500 +0100 @@ -34,6 +34,7 @@ import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.codecs.FieldInfosFormat; +import org.apache.lucene.index.codecs.NormsFormat; import org.apache.lucene.index.codecs.StoredFieldsFormat; import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.SegmentInfosFormat; @@ -41,6 +42,7 @@ import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; import org.apache.lucene.index.codecs.lucene40.Lucene40FieldInfosFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40DocValuesFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40NormsFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40SegmentInfosFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40StoredFieldsFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40TermVectorsFormat; @@ -421,16 +423,20 @@ // auxiliary directory Directory aux = newDirectory(); - setUpDirs(dir, aux); + setUpDirs(dir, aux, true); - IndexReader reader = IndexReader.open(aux, false); + IndexWriterConfig dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMergePolicy(NoMergePolicy.COMPOUND_FILES); + IndexWriter writer = new IndexWriter(aux, dontMergeConfig); for (int i = 0; i < 20; i++) { - reader.deleteDocument(i); + writer.deleteDocuments(new Term("id", "" + i)); } + writer.close(); + IndexReader reader = IndexReader.open(aux); assertEquals(10, reader.numDocs()); reader.close(); - IndexWriter writer = newWriter( + writer = newWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.APPEND). @@ -454,7 +460,7 @@ Directory aux = newDirectory(); Directory aux2 = newDirectory(); - setUpDirs(dir, aux); + setUpDirs(dir, aux, true); IndexWriter writer = newWriter( aux2, @@ -468,17 +474,25 @@ assertEquals(3, writer.getSegmentCount()); writer.close(); - IndexReader reader = IndexReader.open(aux, false); + IndexWriterConfig dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMergePolicy(NoMergePolicy.COMPOUND_FILES); + writer = new IndexWriter(aux, dontMergeConfig); for (int i = 0; i < 27; i++) { - reader.deleteDocument(i); + writer.deleteDocuments(new Term("id", "" + i)); } + writer.close(); + IndexReader reader = IndexReader.open(aux); assertEquals(3, reader.numDocs()); reader.close(); - reader = IndexReader.open(aux2, false); + dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMergePolicy(NoMergePolicy.COMPOUND_FILES); + writer = new IndexWriter(aux2, dontMergeConfig); for (int i = 0; i < 8; i++) { - reader.deleteDocument(i); + writer.deleteDocuments(new Term("id", "" + i)); } + writer.close(); + reader = IndexReader.open(aux2); assertEquals(22, reader.numDocs()); reader.close(); @@ -523,7 +537,7 @@ } private void verifyNumDocs(Directory dir, int numDocs) throws IOException { - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(numDocs, reader.maxDoc()); assertEquals(numDocs, reader.numDocs()); reader.close(); @@ -531,7 +545,7 @@ private void verifyTermDocs(Directory dir, Term term, int numDocs) throws IOException { - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); DocsEnum docsEnum = _TestUtil.docs(random, reader, term.field, term.bytes, null, null, false); int count = 0; while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) @@ -541,11 +555,19 @@ } private void setUpDirs(Directory dir, Directory aux) throws IOException { + setUpDirs(dir, aux, false); + } + + private void setUpDirs(Directory dir, Directory aux, boolean withID) throws IOException { IndexWriter writer = null; writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); // add 1000 documents in 1 segment - addDocs(writer, 1000); + if (withID) { + addDocsWithID(writer, 1000, 0); + } else { + addDocs(writer, 1000); + } assertEquals(1000, writer.maxDoc()); assertEquals(1, writer.getSegmentCount()); writer.close(); @@ -559,7 +581,11 @@ ); // add 30 documents in 3 segments for (int i = 0; i < 3; i++) { - addDocs(writer, 10); + if (withID) { + addDocsWithID(writer, 10, 10*i); + } else { + addDocs(writer, 10); + } writer.close(); writer = newWriter( aux, @@ -657,7 +683,7 @@ readers = new IndexReader[NUM_COPY]; for(int i=0;i= 157); reader.close(); dir.close(); @@ -129,7 +129,7 @@ System.out.println("file " + i + " = " + l[i] + " " + dir.fileLength(l[i]) + " bytes"); */ - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); assertEquals(157, reader.numDocs()); reader.close(); dir.close(); @@ -150,57 +150,9 @@ for(int i=0;i commits) throws IOException { IndexCommit lastCommit = commits.get(commits.size()-1); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); assertEquals("lastCommit.segmentCount()=" + lastCommit.getSegmentCount() + " vs IndexReader.segmentCount=" + r.getSequentialSubReaders().length, r.getSequentialSubReaders().length, lastCommit.getSegmentCount()); r.close(); verifyCommitOrder(commits); @@ -259,7 +260,7 @@ while(gen > 0) { try { - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); reader.close(); fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", @@ -351,7 +352,7 @@ // Make sure we can open a reader on each commit: for (final IndexCommit commit : commits) { - IndexReader r = IndexReader.open(commit, null, false); + IndexReader r = IndexReader.open(commit); r.close(); } @@ -360,7 +361,7 @@ dir.deleteFile(IndexFileNames.SEGMENTS_GEN); long gen = SegmentInfos.getCurrentSegmentGeneration(dir); while(gen > 0) { - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); reader.close(); dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); gen--; @@ -435,7 +436,7 @@ // Should undo our rollback: writer.rollback(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); // Still merged, still 11 docs assertEquals(1, r.getSequentialSubReaders().length); assertEquals(11, r.numDocs()); @@ -450,7 +451,7 @@ // Now 8 because we made another commit assertEquals(7, IndexReader.listCommits(dir).size()); - r = IndexReader.open(dir, true); + r = IndexReader.open(dir); // Not fully merged because we rolled it back, and now only // 10 docs assertTrue(r.getSequentialSubReaders().length > 1); @@ -462,7 +463,7 @@ writer.forceMerge(1); writer.close(); - r = IndexReader.open(dir, true); + r = IndexReader.open(dir); assertEquals(1, r.getSequentialSubReaders().length); assertEquals(10, r.numDocs()); r.close(); @@ -474,7 +475,7 @@ // Reader still sees fully merged index, because writer // opened on the prior commit has not yet committed: - r = IndexReader.open(dir, true); + r = IndexReader.open(dir); assertEquals(1, r.getSequentialSubReaders().length); assertEquals(10, r.numDocs()); r.close(); @@ -482,7 +483,7 @@ writer.close(); // Now reader sees not-fully-merged index: - r = IndexReader.open(dir, true); + r = IndexReader.open(dir); assertTrue(r.getSequentialSubReaders().length > 1); assertEquals(10, r.numDocs()); r.close(); @@ -535,7 +536,7 @@ // Simplistic check: just verify the index is in fact // readable: - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); reader.close(); dir.close(); @@ -583,7 +584,7 @@ long gen = SegmentInfos.getCurrentSegmentGeneration(dir); for(int i=0;i methodsThatShouldNotBeOverridden = new HashSet(); + methodsThatShouldNotBeOverridden.add("doOpenIfChanged"); + methodsThatShouldNotBeOverridden.add("clone"); + boolean fail = false; + for (Method m : FilterIndexReader.class.getMethods()) { + int mods = m.getModifiers(); + if (Modifier.isStatic(mods) || Modifier.isFinal(mods)) { + continue; + } + Class< ? > declaringClass = m.getDeclaringClass(); + String name = m.getName(); + if (declaringClass != FilterIndexReader.class && declaringClass != Object.class && !methodsThatShouldNotBeOverridden.contains(name)) { + System.err.println("method is not overridden by FilterIndexReader: " + name); + fail = true; + } else if (declaringClass == FilterIndexReader.class && methodsThatShouldNotBeOverridden.contains(name)) { + System.err.println("method should not be overridden by FilterIndexReader: " + name); + fail = true; + } + } + assertFalse("FilterIndexReader overrides (or not) some problematic methods; see log above", fail); + } + } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java 2011-11-16 23:30:30.127750500 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java 2011-12-08 19:31:39.023274400 +0100 @@ -68,14 +68,14 @@ writer.close(); // Delete one doc so we get a .del file: - IndexReader reader = IndexReader.open(dir, false); + writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES) + ); Term searchTerm = new Term("id", "7"); - int delCount = reader.deleteDocuments(searchTerm); - assertEquals("didn't delete the right number of documents", 1, delCount); - DefaultSimilarity sim = new DefaultSimilarity(); - // Set one norm so we get a .s0 file: - reader.setNorm(21, "content", sim.encodeNormValue(1.5f)); - reader.close(); + writer.deleteDocuments(searchTerm); + writer.close(); // Now, artificially create an extra .del file & extra // .s0 file: @@ -87,47 +87,6 @@ } */ - // The numbering of fields can vary depending on which - // JRE is in use. On some JREs we see content bound to - // field 0; on others, field 1. So, here we have to - // figure out which field number corresponds to - // "content", and then set our expected file names below - // accordingly: - CompoundFileDirectory cfsReader = new CompoundFileDirectory(dir, "_2.cfs", newIOContext(random), false); - FieldInfosReader infosReader = Codec.getDefault().fieldInfosFormat().getFieldInfosReader(); - FieldInfos fieldInfos = infosReader.read(cfsReader, "2", IOContext.READONCE); - int contentFieldIndex = -1; - for (FieldInfo fi : fieldInfos) { - if (fi.name.equals("content")) { - contentFieldIndex = fi.number; - break; - } - } - cfsReader.close(); - assertTrue("could not locate the 'content' field number in the _2.cfs segment", contentFieldIndex != -1); - - String normSuffix = "s" + contentFieldIndex; - - // Create a bogus separate norms file for a - // segment/field that actually has a separate norms file - // already: - copyFile(dir, "_2_1." + normSuffix, "_2_2." + normSuffix); - - // Create a bogus separate norms file for a - // segment/field that actually has a separate norms file - // already, using the "not compound file" extension: - copyFile(dir, "_2_1." + normSuffix, "_2_2.f" + contentFieldIndex); - - // Create a bogus separate norms file for a - // segment/field that does not have a separate norms - // file already: - copyFile(dir, "_2_1." + normSuffix, "_1_1." + normSuffix); - - // Create a bogus separate norms file for a - // segment/field that does not have a separate norms - // file already using the "not compound file" extension: - copyFile(dir, "_2_1." + normSuffix, "_1_1.f" + contentFieldIndex); - // Create a bogus separate del file for a // segment that already has a separate del file: copyFile(dir, "_0_1.del", "_0_2.del"); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReader.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReader.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReader.java 2011-12-04 20:33:24.324256900 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReader.java 2011-12-08 19:31:37.996215700 +0100 @@ -52,64 +52,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Bits; -public class TestIndexReader extends LuceneTestCase -{ - - public void testCommitUserData() throws Exception { - Directory d = newDirectory(); - - Map commitUserData = new HashMap(); - commitUserData.put("foo", "fighters"); - - // set up writer - IndexWriter writer = new IndexWriter(d, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)) - .setMaxBufferedDocs(2)); - for(int i=0;i<27;i++) - addDocumentWithFields(writer); - writer.close(); - - IndexReader r = IndexReader.open(d, false); - r.deleteDocument(5); - r.flush(commitUserData); - IndexCommit c = r.getIndexCommit(); - r.close(); - - SegmentInfos sis = new SegmentInfos(); - sis.read(d); - IndexReader r2 = IndexReader.open(d, false); - assertEquals(c.getUserData(), commitUserData); - - assertEquals(sis.getCurrentSegmentFileName(), c.getSegmentsFileName()); - - // Change the index - writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode( - OpenMode.APPEND).setMaxBufferedDocs(2)); - for(int i=0;i<7;i++) - addDocumentWithFields(writer); - writer.close(); - - IndexReader r3 = IndexReader.openIfChanged(r2); - assertNotNull(r3); - assertFalse(c.equals(r3.getIndexCommit())); - assertFalse(r2.getIndexCommit().getSegmentCount() == 1 && !r2.hasDeletions()); - r3.close(); - - writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)) - .setOpenMode(OpenMode.APPEND)); - writer.forceMerge(1); - writer.close(); - - r3 = IndexReader.openIfChanged(r2); - assertNotNull(r3); - assertEquals(1, r3.getIndexCommit().getSegmentCount()); - r2.close(); - r3.close(); - d.close(); - } - +public class TestIndexReader extends LuceneTestCase { + public void testIsCurrent() throws Exception { Directory d = newDirectory(); IndexWriter writer = new IndexWriter(d, newIndexWriterConfig( @@ -117,7 +61,7 @@ addDocumentWithFields(writer); writer.close(); // set up reader: - IndexReader reader = IndexReader.open(d, false); + IndexReader reader = IndexReader.open(d); assertTrue(reader.isCurrent()); // modify index by adding another document: writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, @@ -160,7 +104,7 @@ writer.close(); // set up reader - IndexReader reader = IndexReader.open(d, false); + IndexReader reader = IndexReader.open(d); Collection fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL); assertTrue(fieldNames.contains("keyword")); assertTrue(fieldNames.contains("text")); @@ -220,7 +164,7 @@ writer.close(); // verify fields again - reader = IndexReader.open(d, false); + reader = IndexReader.open(d); fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL); assertEquals(13, fieldNames.size()); // the following fields assertTrue(fieldNames.contains("keyword")); @@ -355,7 +299,7 @@ doc.add(new TextField("junk", "junk text")); writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); Document doc2 = reader.document(reader.maxDoc() - 1); IndexableField[] fields = doc2.getFields("bin1"); assertNotNull(fields); @@ -374,7 +318,7 @@ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy())); writer.forceMerge(1); writer.close(); - reader = IndexReader.open(dir, false); + reader = IndexReader.open(dir); doc2 = reader.document(reader.maxDoc() - 1); fields = doc2.getFields("bin1"); assertNotNull(fields); @@ -390,170 +334,6 @@ dir.close(); } - // Make sure attempts to make changes after reader is - // closed throws IOException: - public void testChangesAfterClose() throws IOException { - Directory dir = newDirectory(); - - IndexWriter writer = null; - IndexReader reader = null; - Term searchTerm = new Term("content", "aaa"); - - // add 11 documents with term : aaa - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - for (int i = 0; i < 11; i++) { - addDoc(writer, searchTerm.text()); - } - writer.close(); - - reader = IndexReader.open(dir, false); - - // Close reader: - reader.close(); - - // Then, try to make changes: - try { - reader.deleteDocument(4); - fail("deleteDocument after close failed to throw IOException"); - } catch (AlreadyClosedException e) { - // expected - } - - DefaultSimilarity sim = new DefaultSimilarity(); - try { - reader.setNorm(5, "aaa", sim.encodeNormValue(2.0f)); - fail("setNorm after close failed to throw IOException"); - } catch (AlreadyClosedException e) { - // expected - } - - try { - reader.undeleteAll(); - fail("undeleteAll after close failed to throw IOException"); - } catch (AlreadyClosedException e) { - // expected - } - dir.close(); - } - - // Make sure we get lock obtain failed exception with 2 writers: - public void testLockObtainFailed() throws IOException { - Directory dir = newDirectory(); - - Term searchTerm = new Term("content", "aaa"); - - // add 11 documents with term : aaa - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - writer.commit(); - for (int i = 0; i < 11; i++) { - addDoc(writer, searchTerm.text()); - } - - // Create reader: - IndexReader reader = IndexReader.open(dir, false); - - // Try to make changes - try { - reader.deleteDocument(4); - fail("deleteDocument should have hit LockObtainFailedException"); - } catch (LockObtainFailedException e) { - // expected - } - - DefaultSimilarity sim = new DefaultSimilarity(); - try { - reader.setNorm(5, "aaa", sim.encodeNormValue(2.0f)); - fail("setNorm should have hit LockObtainFailedException"); - } catch (LockObtainFailedException e) { - // expected - } - - try { - reader.undeleteAll(); - fail("undeleteAll should have hit LockObtainFailedException"); - } catch (LockObtainFailedException e) { - // expected - } - writer.close(); - reader.close(); - dir.close(); - } - - // Make sure you can set norms & commit even if a reader - // is open against the index: - public void testWritingNorms() throws IOException { - Directory dir = newDirectory(); - Term searchTerm = new Term("content", "aaa"); - - // add 1 documents with term : aaa - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDoc(writer, searchTerm.text()); - writer.close(); - - // now open reader & set norm for doc 0 - IndexReader reader = IndexReader.open(dir, false); - DefaultSimilarity sim = new DefaultSimilarity(); - reader.setNorm(0, "content", sim.encodeNormValue(2.0f)); - - // we should be holding the write lock now: - assertTrue("locked", IndexWriter.isLocked(dir)); - - reader.commit(); - - // we should not be holding the write lock now: - assertTrue("not locked", !IndexWriter.isLocked(dir)); - - // open a 2nd reader: - IndexReader reader2 = IndexReader.open(dir, false); - - // set norm again for doc 0 - reader.setNorm(0, "content", sim.encodeNormValue(3.0f)); - assertTrue("locked", IndexWriter.isLocked(dir)); - - reader.close(); - - // we should not be holding the write lock now: - assertTrue("not locked", !IndexWriter.isLocked(dir)); - - reader2.close(); - dir.close(); - } - - - // Make sure you can set norms & commit, and there are - // no extra norms files left: - public void testWritingNormsNoReader() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = null; - IndexReader reader = null; - Term searchTerm = new Term("content", "aaa"); - - // add 1 documents with term : aaa - writer = new IndexWriter( - dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setMergePolicy(newLogMergePolicy(false)) - ); - addDoc(writer, searchTerm.text()); - writer.close(); - - DefaultSimilarity sim = new DefaultSimilarity(); - // now open reader & set norm for doc 0 (writes to - // _0_1.s0) - reader = IndexReader.open(dir, false); - reader.setNorm(0, "content", sim.encodeNormValue(2.0f)); - reader.close(); - - // now open reader again & set norm for doc 0 (writes to _0_2.s0) - reader = IndexReader.open(dir, false); - reader.setNorm(0, "content", sim.encodeNormValue(2.0f)); - reader.close(); - assertFalse("failed to remove first generation norms file on writing second generation", - dir.fileExists("_0_1.s0")); - - dir.close(); - } - /* ??? public void testOpenEmptyDirectory() throws IOException{ String dirName = "test.empty"; File fileDirName = new File(dirName); @@ -590,7 +370,7 @@ // Now open existing directory and test that reader closes all files dir = newFSDirectory(dirFile); - IndexReader reader1 = IndexReader.open(dir, false); + IndexReader reader1 = IndexReader.open(dir); reader1.close(); dir.close(); @@ -608,7 +388,7 @@ assertTrue(IndexWriter.isLocked(dir)); // writer open, so dir is locked writer.close(); assertTrue(IndexReader.indexExists(dir)); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); assertFalse(IndexWriter.isLocked(dir)); // reader only, no lock long version = IndexReader.lastModified(dir); if (i == 1) { @@ -623,7 +403,7 @@ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); addDocumentWithFields(writer); writer.close(); - reader = IndexReader.open(dir, false); + reader = IndexReader.open(dir); assertTrue("old lastModified is " + version + "; new lastModified is " + IndexReader.lastModified(dir), version <= IndexReader.lastModified(dir)); reader.close(); dir.close(); @@ -638,7 +418,7 @@ assertTrue(IndexWriter.isLocked(dir)); // writer open, so dir is locked writer.close(); assertTrue(IndexReader.indexExists(dir)); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); assertFalse(IndexWriter.isLocked(dir)); // reader only, no lock long version = IndexReader.getCurrentVersion(dir); reader.close(); @@ -647,114 +427,17 @@ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); addDocumentWithFields(writer); writer.close(); - reader = IndexReader.open(dir, false); + reader = IndexReader.open(dir); assertTrue("old version is " + version + "; new version is " + IndexReader.getCurrentVersion(dir), version < IndexReader.getCurrentVersion(dir)); reader.close(); dir.close(); } - public void testLock() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDocumentWithFields(writer); - writer.close(); - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); - IndexReader reader = IndexReader.open(dir, false); - try { - reader.deleteDocument(0); - fail("expected lock"); - } catch(IOException e) { - // expected exception - } - try { - IndexWriter.unlock(dir); // this should not be done in the real world! - } catch (LockReleaseFailedException lrfe) { - writer.close(); - } - reader.deleteDocument(0); - reader.close(); - writer.close(); - dir.close(); - } - - public void testDocsOutOfOrderJIRA140() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - for(int i=0;i<11;i++) { - addDoc(writer, "aaa"); - } - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - - // Try to delete an invalid docId, yet, within range - // of the final bits of the BitVector: - - boolean gotException = false; - try { - reader.deleteDocument(11); - } catch (ArrayIndexOutOfBoundsException e) { - gotException = true; - } - reader.close(); - - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); - - // We must add more docs to get a new segment written - for(int i=0;i<11;i++) { - addDoc(writer, "aaa"); - } - - // Without the fix for LUCENE-140 this call will - // [incorrectly] hit a "docs out of order" - // IllegalStateException because above out-of-bounds - // deleteDocument corrupted the index: - writer.forceMerge(1); - writer.close(); - if (!gotException) { - fail("delete of out-of-bounds doc number failed to hit exception"); - } - dir.close(); - } - - public void testExceptionReleaseWriteLockJIRA768() throws IOException { - - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDoc(writer, "aaa"); - writer.close(); - - IndexReader reader = IndexReader.open(dir, false); - try { - reader.deleteDocument(1); - fail("did not hit exception when deleting an invalid doc number"); - } catch (ArrayIndexOutOfBoundsException e) { - // expected - } - reader.close(); - if (IndexWriter.isLocked(dir)) { - fail("write lock is still held after close"); - } - - reader = IndexReader.open(dir, false); - DefaultSimilarity sim = new DefaultSimilarity(); - try { - reader.setNorm(1, "content", sim.encodeNormValue(2.0f)); - fail("did not hit exception when calling setNorm on an invalid doc number"); - } catch (ArrayIndexOutOfBoundsException e) { - // expected - } - reader.close(); - if (IndexWriter.isLocked(dir)) { - fail("write lock is still held after close"); - } - dir.close(); - } - public void testOpenReaderAfterDelete() throws IOException { File dirFile = _TestUtil.getTempDir("deletetest"); Directory dir = newFSDirectory(dirFile); try { - IndexReader.open(dir, false); + IndexReader.open(dir); fail("expected FileNotFoundException"); } catch (FileNotFoundException e) { // expected @@ -764,7 +447,7 @@ // Make sure we still get a CorruptIndexException (not NPE): try { - IndexReader.open(dir, false); + IndexReader.open(dir); fail("expected FileNotFoundException"); } catch (FileNotFoundException e) { // expected @@ -946,7 +629,7 @@ SegmentInfos sis = new SegmentInfos(); sis.read(d); - IndexReader r = IndexReader.open(d, false); + IndexReader r = IndexReader.open(d); IndexCommit c = r.getIndexCommit(); assertEquals(sis.getCurrentSegmentFileName(), c.getSegmentsFileName()); @@ -987,96 +670,6 @@ d.close(); } - public void testReadOnly() throws Throwable { - Directory d = newDirectory(); - IndexWriter writer = new IndexWriter(d, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDocumentWithFields(writer); - writer.commit(); - addDocumentWithFields(writer); - writer.close(); - - IndexReader r = IndexReader.open(d, true); - try { - r.deleteDocument(0); - fail(); - } catch (UnsupportedOperationException uoe) { - // expected - } - - writer = new IndexWriter( - d, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setOpenMode(OpenMode.APPEND). - setMergePolicy(newLogMergePolicy(10)) - ); - addDocumentWithFields(writer); - writer.close(); - - // Make sure reopen is still readonly: - IndexReader r2 = IndexReader.openIfChanged(r); - assertNotNull(r2); - r.close(); - - assertFalse(r == r2); - - try { - r2.deleteDocument(0); - fail(); - } catch (UnsupportedOperationException uoe) { - // expected - } - - writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)) - .setOpenMode(OpenMode.APPEND)); - writer.forceMerge(1); - writer.close(); - - // Make sure reopen to a single segment is still readonly: - IndexReader r3 = IndexReader.openIfChanged(r2); - assertNotNull(r3); - assertFalse(r3 == r2); - r2.close(); - - assertFalse(r == r2); - - try { - r3.deleteDocument(0); - fail(); - } catch (UnsupportedOperationException uoe) { - // expected - } - - // Make sure write lock isn't held - writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)) - .setOpenMode(OpenMode.APPEND)); - writer.close(); - - r3.close(); - d.close(); - } - - - // LUCENE-1474 - public void testIndexReader() throws Exception { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - writer.addDocument(createDocument("a")); - writer.addDocument(createDocument("b")); - writer.addDocument(createDocument("c")); - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocuments(new Term("id", "a")); - reader.flush(); - reader.deleteDocuments(new Term("id", "b")); - reader.close(); - IndexReader.open(dir,true).close(); - dir.close(); - } - static Document createDocument(String id) { Document doc = new Document(); FieldType customType = new FieldType(TextField.TYPE_STORED); @@ -1093,7 +686,7 @@ public void testNoDir() throws Throwable { Directory dir = newFSDirectory(_TestUtil.getTempDir("doesnotexist")); try { - IndexReader.open(dir, true); + IndexReader.open(dir); fail("did not hit expected exception"); } catch (NoSuchDirectoryException nsde) { // expected @@ -1138,7 +731,7 @@ writer.close(); // Open reader - IndexReader r = getOnlySegmentReader(IndexReader.open(dir, false)); + IndexReader r = getOnlySegmentReader(IndexReader.open(dir)); final int[] ints = FieldCache.DEFAULT.getInts(r, "number", false); assertEquals(1, ints.length); assertEquals(17, ints[0]); @@ -1173,7 +766,7 @@ writer.commit(); // Open reader1 - IndexReader r = IndexReader.open(dir, false); + IndexReader r = IndexReader.open(dir); IndexReader r1 = getOnlySegmentReader(r); final int[] ints = FieldCache.DEFAULT.getInts(r1, "number", false); assertEquals(1, ints.length); @@ -1207,7 +800,7 @@ writer.addDocument(doc); writer.commit(); - IndexReader r = IndexReader.open(dir, false); + IndexReader r = IndexReader.open(dir); IndexReader r1 = getOnlySegmentReader(r); assertEquals(36, r1.getUniqueTermCount()); writer.addDocument(doc); @@ -1237,7 +830,7 @@ writer.addDocument(doc); writer.close(); - IndexReader r = IndexReader.open(dir, null, true, -1); + IndexReader r = IndexReader.open(dir, -1); try { r.docFreq(new Term("field", "f")); fail("did not hit expected exception"); @@ -1282,7 +875,7 @@ writer.commit(); Document doc = new Document(); writer.addDocument(doc); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); assertTrue(r.isCurrent()); writer.addDocument(doc); writer.prepareCommit(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java 2011-11-12 00:38:46.068905900 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java 2011-12-08 19:31:37.371179900 +0100 @@ -17,14 +17,11 @@ * limitations under the License. */ -import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.Bits; /** * Tests cloning multiple types of readers, modifying the liveDocs and norms @@ -32,468 +29,10 @@ * implemented properly */ public class TestIndexReaderClone extends LuceneTestCase { - - public void testCloneReadOnlySegmentReader() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, false); - IndexReader reader = IndexReader.open(dir1, false); - IndexReader readOnlyReader = reader.clone(true); - if (!isReadOnly(readOnlyReader)) { - fail("reader isn't read only"); - } - if (deleteWorked(1, readOnlyReader)) { - fail("deleting from the original should not have worked"); - } - reader.close(); - readOnlyReader.close(); - dir1.close(); - } - - // open non-readOnly reader1, clone to non-readOnly - // reader2, make sure we can change reader2 - public void testCloneNoChangesStillReadOnly() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, true); - IndexReader r1 = IndexReader.open(dir1, false); - IndexReader r2 = r1.clone(false); - if (!deleteWorked(1, r2)) { - fail("deleting from the cloned should have worked"); - } - r1.close(); - r2.close(); - dir1.close(); - } - - // open non-readOnly reader1, clone to non-readOnly - // reader2, make sure we can change reader1 - public void testCloneWriteToOrig() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, true); - IndexReader r1 = IndexReader.open(dir1, false); - IndexReader r2 = r1.clone(false); - if (!deleteWorked(1, r1)) { - fail("deleting from the original should have worked"); - } - r1.close(); - r2.close(); - dir1.close(); - } - - // open non-readOnly reader1, clone to non-readOnly - // reader2, make sure we can change reader2 - public void testCloneWriteToClone() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, true); - IndexReader r1 = IndexReader.open(dir1, false); - IndexReader r2 = r1.clone(false); - if (!deleteWorked(1, r2)) { - fail("deleting from the original should have worked"); - } - // should fail because reader1 holds the write lock - assertTrue("first reader should not be able to delete", !deleteWorked(1, r1)); - r2.close(); - // should fail because we are now stale (reader1 - // committed changes) - assertTrue("first reader should not be able to delete", !deleteWorked(1, r1)); - r1.close(); - - dir1.close(); - } - - // create single-segment index, open non-readOnly - // SegmentReader, add docs, reopen to multireader, then do - // delete - public void testReopenSegmentReaderToMultiReader() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, false); - IndexReader reader1 = IndexReader.open(dir1, false); - - TestIndexReaderReopen.modifyIndex(5, dir1); - - IndexReader reader2 = IndexReader.openIfChanged(reader1); - assertNotNull(reader2); - assertTrue(reader1 != reader2); - - assertTrue(deleteWorked(1, reader2)); - reader1.close(); - reader2.close(); - dir1.close(); - } - - // open non-readOnly reader1, clone to readOnly reader2 - public void testCloneWriteableToReadOnly() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, true); - IndexReader reader = IndexReader.open(dir1, false); - IndexReader readOnlyReader = reader.clone(true); - if (!isReadOnly(readOnlyReader)) { - fail("reader isn't read only"); - } - if (deleteWorked(1, readOnlyReader)) { - fail("deleting from the original should not have worked"); - } - // this readonly reader shouldn't have a write lock - if (readOnlyReader.hasChanges) { - fail("readOnlyReader has a write lock"); - } - reader.close(); - readOnlyReader.close(); - dir1.close(); - } - - // open non-readOnly reader1, reopen to readOnly reader2 - public void testReopenWriteableToReadOnly() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, true); - IndexReader reader = IndexReader.open(dir1, false); - final int docCount = reader.numDocs(); - assertTrue(deleteWorked(1, reader)); - assertEquals(docCount-1, reader.numDocs()); - - IndexReader readOnlyReader = IndexReader.openIfChanged(reader, true); - assertNotNull(readOnlyReader); - if (!isReadOnly(readOnlyReader)) { - fail("reader isn't read only"); - } - assertFalse(deleteWorked(1, readOnlyReader)); - assertEquals(docCount-1, readOnlyReader.numDocs()); - reader.close(); - readOnlyReader.close(); - dir1.close(); - } - - // open readOnly reader1, clone to non-readOnly reader2 - public void testCloneReadOnlyToWriteable() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, true); - IndexReader reader1 = IndexReader.open(dir1, true); - - IndexReader reader2 = reader1.clone(false); - if (isReadOnly(reader2)) { - fail("reader should not be read only"); - } - assertFalse("deleting from the original reader should not have worked", deleteWorked(1, reader1)); - // this readonly reader shouldn't yet have a write lock - if (reader2.hasChanges) { - fail("cloned reader should not have write lock"); - } - assertTrue("deleting from the cloned reader should have worked", deleteWorked(1, reader2)); - reader1.close(); - reader2.close(); - dir1.close(); - } - - // open non-readOnly reader1 on multi-segment index, then - // fully merge the index, then clone to readOnly reader2 - public void testReadOnlyCloneAfterFullMerge() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, true); - IndexReader reader1 = IndexReader.open(dir1, false); - IndexWriter w = new IndexWriter(dir1, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - w.forceMerge(1); - w.close(); - IndexReader reader2 = reader1.clone(true); - assertTrue(isReadOnly(reader2)); - reader1.close(); - reader2.close(); - dir1.close(); - } - - private static boolean deleteWorked(int doc, IndexReader r) { - boolean exception = false; - try { - // trying to delete from the original reader should throw an exception - r.deleteDocument(doc); - } catch (Exception ex) { - exception = true; - } - return !exception; - } - - public void testCloneReadOnlyDirectoryReader() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, true); - IndexReader reader = IndexReader.open(dir1, false); - IndexReader readOnlyReader = reader.clone(true); - if (!isReadOnly(readOnlyReader)) { - fail("reader isn't read only"); - } - reader.close(); - readOnlyReader.close(); - dir1.close(); - } - - public static boolean isReadOnly(IndexReader r) { - if (r instanceof SegmentReader) { - return ((SegmentReader) r).readOnly; - } else if (r instanceof DirectoryReader) { - return ((DirectoryReader) r).readOnly; - } else { - return false; - } - } - - public void testParallelReader() throws Exception { - final Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, true); - final Directory dir2 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir2, true); - IndexReader r1 = IndexReader.open(dir1, false); - IndexReader r2 = IndexReader.open(dir2, false); - - ParallelReader pr1 = new ParallelReader(); - pr1.add(r1); - pr1.add(r2); - - performDefaultTests(pr1); - pr1.close(); - dir1.close(); - dir2.close(); - } - - /** - * 1. Get a norm from the original reader 2. Clone the original reader 3. - * Delete a document and set the norm of the cloned reader 4. Verify the norms - * are not the same on each reader 5. Verify the doc deleted is only in the - * cloned reader 6. Try to delete a document in the original reader, an - * exception should be thrown - * - * @param r1 IndexReader to perform tests on - * @throws Exception - */ - private void performDefaultTests(IndexReader r1) throws Exception { - DefaultSimilarity sim = new DefaultSimilarity(); - float norm1 = sim.decodeNormValue(MultiNorms.norms(r1, "field1")[4]); - - IndexReader pr1Clone = (IndexReader) r1.clone(); - pr1Clone.deleteDocument(10); - pr1Clone.setNorm(4, "field1", sim.encodeNormValue(0.5f)); - assertTrue(sim.decodeNormValue(MultiNorms.norms(r1, "field1")[4]) == norm1); - assertTrue(sim.decodeNormValue(MultiNorms.norms(pr1Clone, "field1")[4]) != norm1); - - final Bits liveDocs = MultiFields.getLiveDocs(r1); - assertTrue(liveDocs == null || liveDocs.get(10)); - assertFalse(MultiFields.getLiveDocs(pr1Clone).get(10)); - - // try to update the original reader, which should throw an exception - try { - r1.deleteDocument(11); - fail("Tried to delete doc 11 and an exception should have been thrown"); - } catch (Exception exception) { - // expectted - } - pr1Clone.close(); - } - - public void testMixedReaders() throws Exception { - final Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, true); - final Directory dir2 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir2, true); - IndexReader r1 = IndexReader.open(dir1, false); - IndexReader r2 = IndexReader.open(dir2, false); - - MultiReader multiReader = new MultiReader(r1, r2); - performDefaultTests(multiReader); - multiReader.close(); - dir1.close(); - dir2.close(); - } - - public void testSegmentReaderUndeleteall() throws Exception { - final Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, false); - SegmentReader origSegmentReader = getOnlySegmentReader(IndexReader.open(dir1, false)); - origSegmentReader.deleteDocument(10); - assertDelDocsRefCountEquals(1, origSegmentReader); - origSegmentReader.undeleteAll(); - assertNull(origSegmentReader.liveDocsRef); - origSegmentReader.close(); - // need to test norms? - dir1.close(); - } - - public void testSegmentReaderCloseReferencing() throws Exception { - final Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, false); - SegmentReader origSegmentReader = getOnlySegmentReader(IndexReader.open(dir1, false)); - origSegmentReader.deleteDocument(1); - DefaultSimilarity sim = new DefaultSimilarity(); - origSegmentReader.setNorm(4, "field1", sim.encodeNormValue(0.5f)); - - SegmentReader clonedSegmentReader = (SegmentReader) origSegmentReader - .clone(); - assertDelDocsRefCountEquals(2, origSegmentReader); - origSegmentReader.close(); - assertDelDocsRefCountEquals(1, origSegmentReader); - // check the norm refs - SegmentNorms norm = clonedSegmentReader.norms.get("field1"); - assertEquals(1, norm.bytesRef().get()); - clonedSegmentReader.close(); - dir1.close(); - } - - public void testSegmentReaderDelDocsReferenceCounting() throws Exception { - final Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, false); - - IndexReader origReader = IndexReader.open(dir1, false); - SegmentReader origSegmentReader = getOnlySegmentReader(origReader); - // liveDocsRef should be null because nothing has updated yet - assertNull(origSegmentReader.liveDocsRef); - - // we deleted a document, so there is now a liveDocs bitvector and a - // reference to it - origReader.deleteDocument(1); - assertDelDocsRefCountEquals(1, origSegmentReader); - - // the cloned segmentreader should have 2 references, 1 to itself, and 1 to - // the original segmentreader - IndexReader clonedReader = (IndexReader) origReader.clone(); - SegmentReader clonedSegmentReader = getOnlySegmentReader(clonedReader); - assertDelDocsRefCountEquals(2, origSegmentReader); - // deleting a document creates a new liveDocs bitvector, the refs goes to - // 1 - clonedReader.deleteDocument(2); - assertDelDocsRefCountEquals(1, origSegmentReader); - assertDelDocsRefCountEquals(1, clonedSegmentReader); - - // make sure the deletedocs objects are different (copy - // on write) - assertTrue(origSegmentReader.liveDocs != clonedSegmentReader.liveDocs); - - assertDocDeleted(origSegmentReader, clonedSegmentReader, 1); - final Bits liveDocs = origSegmentReader.getLiveDocs(); - assertTrue(liveDocs == null || liveDocs.get(2)); // doc 2 should not be deleted - // in original segmentreader - assertFalse(clonedSegmentReader.getLiveDocs().get(2)); // doc 2 should be deleted in - // cloned segmentreader - - // deleting a doc from the original segmentreader should throw an exception - try { - origReader.deleteDocument(4); - fail("expected exception"); - } catch (LockObtainFailedException lbfe) { - // expected - } - - origReader.close(); - // try closing the original segment reader to see if it affects the - // clonedSegmentReader - clonedReader.deleteDocument(3); - clonedReader.flush(); - assertDelDocsRefCountEquals(1, clonedSegmentReader); - - // test a reopened reader - IndexReader reopenedReader = IndexReader.openIfChanged(clonedReader); - if (reopenedReader == null) { - reopenedReader = clonedReader; - } - IndexReader cloneReader2 = (IndexReader) reopenedReader.clone(); - SegmentReader cloneSegmentReader2 = getOnlySegmentReader(cloneReader2); - assertDelDocsRefCountEquals(2, cloneSegmentReader2); - clonedReader.close(); - reopenedReader.close(); - cloneReader2.close(); - - dir1.close(); - } - - // LUCENE-1648 - public void testCloneWithDeletes() throws Throwable { - final Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, false); - IndexReader origReader = IndexReader.open(dir1, false); - origReader.deleteDocument(1); - - IndexReader clonedReader = (IndexReader) origReader.clone(); - origReader.close(); - clonedReader.close(); - - IndexReader r = IndexReader.open(dir1, false); - assertFalse(MultiFields.getLiveDocs(r).get(1)); - r.close(); - dir1.close(); - } - - // LUCENE-1648 - public void testCloneWithSetNorm() throws Throwable { - final Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, false); - IndexReader orig = IndexReader.open(dir1, false); - DefaultSimilarity sim = new DefaultSimilarity(); - orig.setNorm(1, "field1", sim.encodeNormValue(17.0f)); - final byte encoded = sim.encodeNormValue(17.0f); - assertEquals(encoded, MultiNorms.norms(orig, "field1")[1]); - - // the cloned segmentreader should have 2 references, 1 to itself, and 1 to - // the original segmentreader - IndexReader clonedReader = (IndexReader) orig.clone(); - orig.close(); - clonedReader.close(); - - IndexReader r = IndexReader.open(dir1, false); - assertEquals(encoded, MultiNorms.norms(r, "field1")[1]); - r.close(); - dir1.close(); - } - - private void assertDocDeleted(SegmentReader reader, SegmentReader reader2, - int doc) { - assertEquals(reader.getLiveDocs().get(doc), reader2.getLiveDocs().get(doc)); - } private void assertDelDocsRefCountEquals(int refCount, SegmentReader reader) { assertEquals(refCount, reader.liveDocsRef.get()); } - - public void testCloneSubreaders() throws Exception { - final Directory dir1 = newDirectory(); - - TestIndexReaderReopen.createIndex(random, dir1, true); - IndexReader reader = IndexReader.open(dir1, false); - reader.deleteDocument(1); // acquire write lock - IndexReader[] subs = reader.getSequentialSubReaders(); - assert subs.length > 1; - - IndexReader[] clones = new IndexReader[subs.length]; - for (int x=0; x < subs.length; x++) { - clones[x] = (IndexReader) subs[x].clone(); - } - reader.close(); - for (int x=0; x < subs.length; x++) { - clones[x].close(); - } - dir1.close(); - } - - public void testLucene1516Bug() throws Exception { - final Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, false); - IndexReader r1 = IndexReader.open(dir1, false); - r1.incRef(); - IndexReader r2 = r1.clone(false); - r1.deleteDocument(5); - r1.decRef(); - - r1.incRef(); - - r2.close(); - r1.decRef(); - r1.close(); - dir1.close(); - } public void testCloseStoredFields() throws Exception { final Directory dir = newDirectory(); @@ -506,8 +45,8 @@ doc.add(newField("field", "yes it's stored", TextField.TYPE_STORED)); w.addDocument(doc); w.close(); - IndexReader r1 = IndexReader.open(dir, false); - IndexReader r2 = r1.clone(false); + IndexReader r1 = IndexReader.open(dir); + IndexReader r2 = (IndexReader) r1.clone(); r1.close(); r2.close(); dir.close(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java 2011-11-12 00:38:45.616505100 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,362 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Random; - -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.search.similarities.DefaultSimilarityProvider; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarities.SimilarityProvider; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; - -/** - * Tests cloning IndexReader norms - */ -public class TestIndexReaderCloneNorms extends LuceneTestCase { - - private class SimilarityProviderOne extends DefaultSimilarityProvider { - @Override - public Similarity get(String field) { - return new DefaultSimilarity() { - @Override - public byte computeNorm(FieldInvertState state) { - // diable length norm - return encodeNormValue(state.getBoost()); - } - }; - } - } - - private static final int NUM_FIELDS = 10; - - private SimilarityProvider similarityProviderOne; - - private Analyzer anlzr; - - private int numDocNorms; - - private ArrayList norms; - - private ArrayList modifiedNorms; - - private float lastNorm = 0; - - private float normDelta = (float) 0.001; - - @Override - public void setUp() throws Exception { - super.setUp(); - similarityProviderOne = new SimilarityProviderOne(); - anlzr = new MockAnalyzer(random); - } - - /** - * Test that norms values are preserved as the index is maintained. Including - * separate norms. Including merging indexes with seprate norms. Including - * full merge. - */ - public void testNorms() throws IOException { - // test with a single index: index1 - Directory dir1 = newDirectory(); - IndexWriter.unlock(dir1); - - norms = new ArrayList(); - modifiedNorms = new ArrayList(); - - createIndex(random, dir1); - doTestNorms(random, dir1); - - // test with a single index: index2 - ArrayList norms1 = norms; - ArrayList modifiedNorms1 = modifiedNorms; - int numDocNorms1 = numDocNorms; - - norms = new ArrayList(); - modifiedNorms = new ArrayList(); - numDocNorms = 0; - - Directory dir2 = newDirectory(); - - createIndex(random, dir2); - doTestNorms(random, dir2); - - // add index1 and index2 to a third index: index3 - Directory dir3 = newDirectory(); - - createIndex(random, dir3); - if (VERBOSE) { - System.out.println("TEST: now addIndexes/full merge"); - } - IndexWriter iw = new IndexWriter( - dir3, - newIndexWriterConfig(TEST_VERSION_CURRENT, anlzr). - setOpenMode(OpenMode.APPEND). - setMaxBufferedDocs(5). - setMergePolicy(newLogMergePolicy(3)) - ); - iw.addIndexes(dir1, dir2); - iw.forceMerge(1); - iw.close(); - - norms1.addAll(norms); - norms = norms1; - modifiedNorms1.addAll(modifiedNorms); - modifiedNorms = modifiedNorms1; - numDocNorms += numDocNorms1; - - // test with index3 - verifyIndex(dir3); - doTestNorms(random, dir3); - - // now with full merge - iw = new IndexWriter( - dir3, - newIndexWriterConfig(TEST_VERSION_CURRENT, anlzr). - setOpenMode(OpenMode.APPEND). - setMaxBufferedDocs(5). - setMergePolicy(newLogMergePolicy(3)) - ); - iw.forceMerge(1); - iw.close(); - verifyIndex(dir3); - - dir1.close(); - dir2.close(); - dir3.close(); - } - - // try cloning and reopening the norms - private void doTestNorms(Random random, Directory dir) throws IOException { - if (VERBOSE) { - System.out.println("TEST: now doTestNorms"); - } - addDocs(random, dir, 12, true); - IndexReader ir = IndexReader.open(dir, false); - verifyIndex(ir); - modifyNormsForF1(ir); - IndexReader irc = (IndexReader) ir.clone();// IndexReader.open(dir, false);//ir.clone(); - verifyIndex(irc); - - modifyNormsForF1(irc); - - IndexReader irc3 = (IndexReader) irc.clone(); - verifyIndex(irc3); - modifyNormsForF1(irc3); - verifyIndex(irc3); - irc3.flush(); - - ir.close(); - irc.close(); - irc3.close(); - } - - public void testNormsClose() throws IOException { - Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, false); - SegmentReader reader1 = getOnlySegmentReader(IndexReader.open(dir1, false)); - reader1.norms("field1"); - SegmentNorms r1norm = reader1.norms.get("field1"); - AtomicInteger r1BytesRef = r1norm.bytesRef(); - SegmentReader reader2 = (SegmentReader)reader1.clone(); - assertEquals(2, r1norm.bytesRef().get()); - reader1.close(); - assertEquals(1, r1BytesRef.get()); - reader2.norms("field1"); - reader2.close(); - dir1.close(); - } - - public void testNormsRefCounting() throws IOException { - Directory dir1 = newDirectory(); - TestIndexReaderReopen.createIndex(random, dir1, false); - IndexReader reader1 = IndexReader.open(dir1, false); - - IndexReader reader2C = (IndexReader) reader1.clone(); - SegmentReader segmentReader2C = getOnlySegmentReader(reader2C); - segmentReader2C.norms("field1"); // load the norms for the field - SegmentNorms reader2CNorm = segmentReader2C.norms.get("field1"); - assertTrue("reader2CNorm.bytesRef()=" + reader2CNorm.bytesRef(), reader2CNorm.bytesRef().get() == 2); - - - - IndexReader reader3C = (IndexReader) reader2C.clone(); - SegmentReader segmentReader3C = getOnlySegmentReader(reader3C); - SegmentNorms reader3CCNorm = segmentReader3C.norms.get("field1"); - assertEquals(3, reader3CCNorm.bytesRef().get()); - - // edit a norm and the refcount should be 1 - IndexReader reader4C = (IndexReader) reader3C.clone(); - SegmentReader segmentReader4C = getOnlySegmentReader(reader4C); - assertEquals(4, reader3CCNorm.bytesRef().get()); - DefaultSimilarity sim = new DefaultSimilarity(); - reader4C.setNorm(5, "field1", sim.encodeNormValue(0.33f)); - - // generate a cannot update exception in reader1 - try { - reader3C.setNorm(1, "field1", sim.encodeNormValue(0.99f)); - fail("did not hit expected exception"); - } catch (Exception ex) { - // expected - } - - // norm values should be different - assertTrue(sim.decodeNormValue(segmentReader3C.norms("field1")[5]) - != sim.decodeNormValue(segmentReader4C.norms("field1")[5])); - SegmentNorms reader4CCNorm = segmentReader4C.norms.get("field1"); - assertEquals(3, reader3CCNorm.bytesRef().get()); - assertEquals(1, reader4CCNorm.bytesRef().get()); - - IndexReader reader5C = (IndexReader) reader4C.clone(); - SegmentReader segmentReader5C = getOnlySegmentReader(reader5C); - SegmentNorms reader5CCNorm = segmentReader5C.norms.get("field1"); - reader5C.setNorm(5, "field1", sim.encodeNormValue(0.7f)); - assertEquals(1, reader5CCNorm.bytesRef().get()); - - reader5C.close(); - reader4C.close(); - reader3C.close(); - reader2C.close(); - reader1.close(); - dir1.close(); - } - - private void createIndex(Random random, Directory dir) throws IOException { - if (VERBOSE) { - System.out.println("TEST: createIndex"); - } - IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.CREATE) - .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newLogMergePolicy())); - - LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy(); - lmp.setMergeFactor(3); - lmp.setUseCompoundFile(true); - iw.close(); - if (VERBOSE) { - System.out.println("TEST: done createIndex"); - } - } - - private void modifyNormsForF1(IndexReader ir) throws IOException { - int n = ir.maxDoc(); - // System.out.println("modifyNormsForF1 maxDoc: "+n); - for (int i = 0; i < n; i += 3) { // modify for every third doc - int k = (i * 3) % modifiedNorms.size(); - float origNorm = modifiedNorms.get(i).floatValue(); - float newNorm = modifiedNorms.get(k).floatValue(); - // System.out.println("Modifying: for "+i+" from "+origNorm+" to - // "+newNorm); - // System.out.println(" and: for "+k+" from "+newNorm+" to "+origNorm); - modifiedNorms.set(i, Float.valueOf(newNorm)); - modifiedNorms.set(k, Float.valueOf(origNorm)); - DefaultSimilarity sim = new DefaultSimilarity(); - ir.setNorm(i, "f" + 1, sim.encodeNormValue(newNorm)); - ir.setNorm(k, "f" + 1, sim.encodeNormValue(origNorm)); - // System.out.println("setNorm i: "+i); - // break; - } - // ir.close(); - } - - private void verifyIndex(Directory dir) throws IOException { - IndexReader ir = IndexReader.open(dir, false); - verifyIndex(ir); - ir.close(); - } - - private void verifyIndex(IndexReader ir) throws IOException { - for (int i = 0; i < NUM_FIELDS; i++) { - String field = "f" + i; - byte b[] = MultiNorms.norms(ir, field); - assertEquals("number of norms mismatches", numDocNorms, b.length); - ArrayList storedNorms = (i == 1 ? modifiedNorms : norms); - for (int j = 0; j < b.length; j++) { - DefaultSimilarity sim = new DefaultSimilarity(); - float norm = sim.decodeNormValue(b[j]); - float norm1 = storedNorms.get(j).floatValue(); - assertEquals("stored norm value of " + field + " for doc " + j + " is " - + norm + " - a mismatch!", norm, norm1, 0.000001); - } - } - } - - private void addDocs(Random random, Directory dir, int ndocs, boolean compound) - throws IOException { - IndexWriterConfig conf = newIndexWriterConfig( - TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND) - .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newLogMergePolicy()); - LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); - lmp.setMergeFactor(3); - lmp.setUseCompoundFile(compound); - IndexWriter iw = new IndexWriter(dir, conf); - for (int i = 0; i < ndocs; i++) { - iw.addDocument(newDoc()); - } - iw.close(); - } - - // create the next document - private Document newDoc() { - Document d = new Document(); - float boost = nextNorm("anyfield"); // in this test the same similarity is used for all fields so it does not matter what field is passed - - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); - customType.setTokenized(false); - for (int i = 0; i < 10; i++) { - Field f = newField("f" + i, "v" + i, customType); - f.setBoost(boost); - d.add(f); - } - return d; - } - - // return unique norm values that are unchanged by encoding/decoding - private float nextNorm(String fname) { - float norm = lastNorm + normDelta; - DefaultSimilarity sim = new DefaultSimilarity(); - do { - float norm1 = sim.decodeNormValue( - sim.encodeNormValue(norm)); - if (norm1 > lastNorm) { - // System.out.println(norm1+" > "+lastNorm); - norm = norm1; - break; - } - norm += normDelta; - } while (true); - norms.add(numDocNorms, Float.valueOf(norm)); - modifiedNorms.add(numDocNorms, Float.valueOf(norm)); - // System.out.println("creating norm("+numDocNorms+"): "+norm); - numDocNorms++; - lastNorm = (norm > 10 ? 0 : norm); // there's a limit to how many distinct - // values can be stored in a ingle byte - return norm; - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java 2011-11-12 00:38:45.663305200 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,375 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.util.LuceneTestCase; - -import static org.apache.lucene.index.TestIndexReader.addDoc; -import static org.apache.lucene.index.TestIndexReader.addDocumentWithFields; -import static org.apache.lucene.index.TestIndexReader.assertTermDocsCount; -import static org.apache.lucene.index.TestIndexReader.createDocument; - -public class TestIndexReaderDelete extends LuceneTestCase { - private void deleteReaderReaderConflict(boolean doFullMerge) throws IOException { - Directory dir = newDirectory(); - - Term searchTerm1 = new Term("content", "aaa"); - Term searchTerm2 = new Term("content", "bbb"); - Term searchTerm3 = new Term("content", "ccc"); - - // add 100 documents with term : aaa - // add 100 documents with term : bbb - // add 100 documents with term : ccc - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); - for (int i = 0; i < 100; i++) { - addDoc(writer, searchTerm1.text()); - addDoc(writer, searchTerm2.text()); - addDoc(writer, searchTerm3.text()); - } - if (doFullMerge) { - writer.forceMerge(1); - } - writer.close(); - - // OPEN TWO READERS - // Both readers get segment info as exists at this time - IndexReader reader1 = IndexReader.open(dir, false); - assertEquals("first opened", 100, reader1.docFreq(searchTerm1)); - assertEquals("first opened", 100, reader1.docFreq(searchTerm2)); - assertEquals("first opened", 100, reader1.docFreq(searchTerm3)); - assertTermDocsCount("first opened", reader1, searchTerm1, 100); - assertTermDocsCount("first opened", reader1, searchTerm2, 100); - assertTermDocsCount("first opened", reader1, searchTerm3, 100); - - IndexReader reader2 = IndexReader.open(dir, false); - assertEquals("first opened", 100, reader2.docFreq(searchTerm1)); - assertEquals("first opened", 100, reader2.docFreq(searchTerm2)); - assertEquals("first opened", 100, reader2.docFreq(searchTerm3)); - assertTermDocsCount("first opened", reader2, searchTerm1, 100); - assertTermDocsCount("first opened", reader2, searchTerm2, 100); - assertTermDocsCount("first opened", reader2, searchTerm3, 100); - - // DELETE DOCS FROM READER 2 and CLOSE IT - // delete documents containing term: aaa - // when the reader is closed, the segment info is updated and - // the first reader is now stale - reader2.deleteDocuments(searchTerm1); - assertEquals("after delete 1", 100, reader2.docFreq(searchTerm1)); - assertEquals("after delete 1", 100, reader2.docFreq(searchTerm2)); - assertEquals("after delete 1", 100, reader2.docFreq(searchTerm3)); - assertTermDocsCount("after delete 1", reader2, searchTerm1, 0); - assertTermDocsCount("after delete 1", reader2, searchTerm2, 100); - assertTermDocsCount("after delete 1", reader2, searchTerm3, 100); - reader2.close(); - - // Make sure reader 1 is unchanged since it was open earlier - assertEquals("after delete 1", 100, reader1.docFreq(searchTerm1)); - assertEquals("after delete 1", 100, reader1.docFreq(searchTerm2)); - assertEquals("after delete 1", 100, reader1.docFreq(searchTerm3)); - assertTermDocsCount("after delete 1", reader1, searchTerm1, 100); - assertTermDocsCount("after delete 1", reader1, searchTerm2, 100); - assertTermDocsCount("after delete 1", reader1, searchTerm3, 100); - - - // ATTEMPT TO DELETE FROM STALE READER - // delete documents containing term: bbb - try { - reader1.deleteDocuments(searchTerm2); - fail("Delete allowed from a stale index reader"); - } catch (IOException e) { - /* success */ - } - - // RECREATE READER AND TRY AGAIN - reader1.close(); - reader1 = IndexReader.open(dir, false); - assertEquals("reopened", 100, reader1.docFreq(searchTerm1)); - assertEquals("reopened", 100, reader1.docFreq(searchTerm2)); - assertEquals("reopened", 100, reader1.docFreq(searchTerm3)); - assertTermDocsCount("reopened", reader1, searchTerm1, 0); - assertTermDocsCount("reopened", reader1, searchTerm2, 100); - assertTermDocsCount("reopened", reader1, searchTerm3, 100); - - reader1.deleteDocuments(searchTerm2); - assertEquals("deleted 2", 100, reader1.docFreq(searchTerm1)); - assertEquals("deleted 2", 100, reader1.docFreq(searchTerm2)); - assertEquals("deleted 2", 100, reader1.docFreq(searchTerm3)); - assertTermDocsCount("deleted 2", reader1, searchTerm1, 0); - assertTermDocsCount("deleted 2", reader1, searchTerm2, 0); - assertTermDocsCount("deleted 2", reader1, searchTerm3, 100); - reader1.close(); - - // Open another reader to confirm that everything is deleted - reader2 = IndexReader.open(dir, false); - assertTermDocsCount("reopened 2", reader2, searchTerm1, 0); - assertTermDocsCount("reopened 2", reader2, searchTerm2, 0); - assertTermDocsCount("reopened 2", reader2, searchTerm3, 100); - reader2.close(); - - dir.close(); - } - - private void deleteReaderWriterConflict(boolean doFullMerge) throws IOException { - //Directory dir = new RAMDirectory(); - Directory dir = newDirectory(); - - Term searchTerm = new Term("content", "aaa"); - Term searchTerm2 = new Term("content", "bbb"); - - // add 100 documents with term : aaa - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); - for (int i = 0; i < 100; i++) { - addDoc(writer, searchTerm.text()); - } - writer.close(); - - // OPEN READER AT THIS POINT - this should fix the view of the - // index at the point of having 100 "aaa" documents and 0 "bbb" - IndexReader reader = IndexReader.open(dir, false); - assertEquals("first docFreq", 100, reader.docFreq(searchTerm)); - assertEquals("first docFreq", 0, reader.docFreq(searchTerm2)); - assertTermDocsCount("first reader", reader, searchTerm, 100); - assertTermDocsCount("first reader", reader, searchTerm2, 0); - - // add 100 documents with term : bbb - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); - for (int i = 0; i < 100; i++) { - addDoc(writer, searchTerm2.text()); - } - - // REQUEST full merge - // This causes a new segment to become current for all subsequent - // searchers. Because of this, deletions made via a previously open - // reader, which would be applied to that reader's segment, are lost - // for subsequent searchers/readers - if (doFullMerge) { - writer.forceMerge(1); - } - writer.close(); - - // The reader should not see the new data - assertEquals("first docFreq", 100, reader.docFreq(searchTerm)); - assertEquals("first docFreq", 0, reader.docFreq(searchTerm2)); - assertTermDocsCount("first reader", reader, searchTerm, 100); - assertTermDocsCount("first reader", reader, searchTerm2, 0); - - - // DELETE DOCUMENTS CONTAINING TERM: aaa - // NOTE: the reader was created when only "aaa" documents were in - int deleted = 0; - try { - deleted = reader.deleteDocuments(searchTerm); - fail("Delete allowed on an index reader with stale segment information"); - } catch (StaleReaderException e) { - /* success */ - } - - // Re-open index reader and try again. This time it should see - // the new data. - reader.close(); - reader = IndexReader.open(dir, false); - assertEquals("first docFreq", 100, reader.docFreq(searchTerm)); - assertEquals("first docFreq", 100, reader.docFreq(searchTerm2)); - assertTermDocsCount("first reader", reader, searchTerm, 100); - assertTermDocsCount("first reader", reader, searchTerm2, 100); - - deleted = reader.deleteDocuments(searchTerm); - assertEquals("deleted count", 100, deleted); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm)); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2)); - assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); - assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100); - reader.close(); - - // CREATE A NEW READER and re-test - reader = IndexReader.open(dir, false); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2)); - assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); - assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100); - reader.close(); - dir.close(); - } - - public void testBasicDelete() throws IOException { - Directory dir = newDirectory(); - - IndexWriter writer = null; - IndexReader reader = null; - Term searchTerm = new Term("content", "aaa"); - - // add 100 documents with term : aaa - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - for (int i = 0; i < 100; i++) { - addDoc(writer, searchTerm.text()); - } - writer.close(); - - // OPEN READER AT THIS POINT - this should fix the view of the - // index at the point of having 100 "aaa" documents and 0 "bbb" - reader = IndexReader.open(dir, false); - assertEquals("first docFreq", 100, reader.docFreq(searchTerm)); - assertTermDocsCount("first reader", reader, searchTerm, 100); - reader.close(); - - // DELETE DOCUMENTS CONTAINING TERM: aaa - int deleted = 0; - reader = IndexReader.open(dir, false); - deleted = reader.deleteDocuments(searchTerm); - assertEquals("deleted count", 100, deleted); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm)); - assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); - - // open a 2nd reader to make sure first reader can - // commit its changes (.del) while second reader - // is open: - IndexReader reader2 = IndexReader.open(dir, false); - reader.close(); - - // CREATE A NEW READER and re-test - reader = IndexReader.open(dir, false); - assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm)); - assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); - reader.close(); - reader2.close(); - dir.close(); - } - - public void testDeleteReaderReaderConflictNoFullMerge() throws IOException { - deleteReaderReaderConflict(false); - } - - public void testDeleteReaderReaderConflictFullMerge() throws IOException { - deleteReaderReaderConflict(true); - } - - public void testDeleteReaderWriterConflictNoFullMerge() throws IOException { - deleteReaderWriterConflict(false); - } - - public void testDeleteReaderWriterConflictFullMerge() throws IOException { - deleteReaderWriterConflict(true); - } - - public void testMultiReaderDeletes() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter w= new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); - Document doc = new Document(); - doc.add(newField("f", "doctor", StringField.TYPE_UNSTORED)); - w.addDocument(doc); - doc = new Document(); - w.commit(); - doc.add(newField("f", "who", StringField.TYPE_UNSTORED)); - w.addDocument(doc); - IndexReader r = new SlowMultiReaderWrapper(w.getReader()); - w.close(); - - assertNull(r.getLiveDocs()); - r.close(); - - r = new SlowMultiReaderWrapper(IndexReader.open(dir, false)); - - assertNull(r.getLiveDocs()); - assertEquals(1, r.deleteDocuments(new Term("f", "doctor"))); - assertNotNull(r.getLiveDocs()); - assertFalse(r.getLiveDocs().get(0)); - assertEquals(1, r.deleteDocuments(new Term("f", "who"))); - assertFalse(r.getLiveDocs().get(1)); - r.close(); - dir.close(); - } - - public void testUndeleteAll() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDocumentWithFields(writer); - addDocumentWithFields(writer); - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocument(0); - reader.deleteDocument(1); - reader.undeleteAll(); - reader.close(); - reader = IndexReader.open(dir, false); - assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll() - reader.close(); - dir.close(); - } - - public void testUndeleteAllAfterClose() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDocumentWithFields(writer); - addDocumentWithFields(writer); - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocument(0); - reader.close(); - reader = IndexReader.open(dir, false); - reader.undeleteAll(); - assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll() - reader.close(); - dir.close(); - } - - public void testUndeleteAllAfterCloseThenReopen() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - addDocumentWithFields(writer); - addDocumentWithFields(writer); - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocument(0); - reader.close(); - reader = IndexReader.open(dir, false); - reader.undeleteAll(); - reader.close(); - reader = IndexReader.open(dir, false); - assertEquals(2, reader.numDocs()); // nothing has really been deleted thanks to undeleteAll() - reader.close(); - dir.close(); - } - - // LUCENE-1647 - public void testIndexReaderUnDeleteAll() throws Exception { - MockDirectoryWrapper dir = newDirectory(); - dir.setPreventDoubleWrite(false); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - writer.addDocument(createDocument("a")); - writer.addDocument(createDocument("b")); - writer.addDocument(createDocument("c")); - writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocuments(new Term("id", "a")); - reader.flush(); - reader.deleteDocuments(new Term("id", "b")); - reader.undeleteAll(); - reader.deleteDocuments(new Term("id", "b")); - reader.close(); - IndexReader.open(dir,true).close(); - dir.close(); - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java 2011-11-10 16:27:07.974717900 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,229 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util._TestUtil; - -public class TestIndexReaderOnDiskFull extends LuceneTestCase { - /** - * Make sure if reader tries to commit but hits disk - * full that reader remains consistent and usable. - */ - public void testDiskFull() throws IOException { - - Term searchTerm = new Term("content", "aaa"); - int START_COUNT = 157; - int END_COUNT = 144; - - // First build up a starting index: - MockDirectoryWrapper startDir = newDirectory(); - IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - if (VERBOSE) { - System.out.println("TEST: create initial index"); - } - for(int i=0;i<157;i++) { - Document d = new Document(); - d.add(newField("id", Integer.toString(i), StringField.TYPE_STORED)); - d.add(newField("content", "aaa " + i, TextField.TYPE_UNSTORED)); - writer.addDocument(d); - if (0==i%10) - writer.commit(); - } - writer.close(); - - { - IndexReader r = IndexReader.open(startDir); - IndexSearcher searcher = newSearcher(r); - ScoreDoc[] hits = null; - try { - hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; - } catch (IOException e) { - e.printStackTrace(); - fail("exception when init searching: " + e); - } - searcher.close(); - r.close(); - } - - long diskUsage = startDir.getRecomputedActualSizeInBytes(); - long diskFree = diskUsage+_TestUtil.nextInt(random, 50, 200); - - IOException err = null; - - boolean done = false; - boolean gotExc = false; - - // Iterate w/ ever increasing free disk space: - while(!done) { - MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir, newIOContext(random))); - - // If IndexReader hits disk full, it can write to - // the same files again. - dir.setPreventDoubleWrite(false); - - IndexReader reader = IndexReader.open(dir, false); - - // For each disk size, first try to commit against - // dir that will hit random IOExceptions & disk - // full; after, give it infinite disk space & turn - // off random IOExceptions & retry w/ same reader: - boolean success = false; - - for(int x=0;x<2;x++) { - - double rate = 0.05; - double diskRatio = ((double) diskFree)/diskUsage; - long thisDiskFree; - String testName; - - if (0 == x) { - thisDiskFree = diskFree; - if (diskRatio >= 2.0) { - rate /= 2; - } - if (diskRatio >= 4.0) { - rate /= 2; - } - if (diskRatio >= 6.0) { - rate = 0.0; - } - if (VERBOSE) { - System.out.println("\ncycle: " + diskFree + " bytes"); - } - testName = "disk full during reader.close() @ " + thisDiskFree + " bytes"; - } else { - thisDiskFree = 0; - rate = 0.0; - if (VERBOSE) { - System.out.println("\ncycle: same writer: unlimited disk space"); - } - testName = "reader re-use after disk full"; - } - - dir.setMaxSizeInBytes(thisDiskFree); - dir.setRandomIOExceptionRate(rate); - DefaultSimilarity sim = new DefaultSimilarity(); - try { - if (0 == x) { - int docId = 12; - for(int i=0;i<13;i++) { - reader.deleteDocument(docId); - reader.setNorm(docId, "content", sim.encodeNormValue(2.0f)); - docId += 12; - } - } - reader.close(); - success = true; - if (0 == x) { - done = true; - } - } catch (IOException e) { - if (VERBOSE) { - System.out.println(" hit IOException: " + e); - e.printStackTrace(System.out); - } - err = e; - gotExc = true; - if (1 == x) { - e.printStackTrace(); - fail(testName + " hit IOException after disk space was freed up"); - } - } - - // Finally, verify index is not corrupt, and, if - // we succeeded, we see all docs changed, and if - // we failed, we see either all docs or no docs - // changed (transactional semantics): - IndexReader newReader = null; - try { - newReader = IndexReader.open(dir, false); - } catch (IOException e) { - e.printStackTrace(); - fail(testName + ":exception when creating IndexReader after disk full during close: " + e); - } - /* - int result = newReader.docFreq(searchTerm); - if (success) { - if (result != END_COUNT) { - fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + END_COUNT); - } - } else { - // On hitting exception we still may have added - // all docs: - if (result != START_COUNT && result != END_COUNT) { - err.printStackTrace(); - fail(testName + ": method did throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " or " + END_COUNT); - } - } - */ - - IndexSearcher searcher = newSearcher(newReader); - ScoreDoc[] hits = null; - try { - hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; - } catch (IOException e) { - e.printStackTrace(); - fail(testName + ": exception when searching: " + e); - } - int result2 = hits.length; - if (success) { - if (result2 != END_COUNT) { - fail(testName + ": method did not throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + END_COUNT); - } - } else { - // On hitting exception we still may have added - // all docs: - if (result2 != START_COUNT && result2 != END_COUNT) { - err.printStackTrace(); - fail(testName + ": method did throw exception but hits.length for search on term 'aaa' is " + result2 + " instead of expected " + START_COUNT); - } - } - - searcher.close(); - newReader.close(); - - if (result2 == END_COUNT) { - if (!gotExc) - fail("never hit disk full"); - break; - } - } - - dir.close(); - - // Try again with more bytes of free space: - diskFree += TEST_NIGHTLY ? _TestUtil.nextInt(random, 5, 20) : _TestUtil.nextInt(random, 50, 200); - } - - startDir.close(); - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java 2011-11-12 00:38:46.256106200 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java 2011-12-08 19:31:41.053390500 +0100 @@ -62,7 +62,7 @@ @Override protected IndexReader openReader() throws IOException { - return IndexReader.open(dir1, false); + return IndexReader.open(dir1); } }); @@ -80,7 +80,7 @@ @Override protected IndexReader openReader() throws IOException { - return IndexReader.open(dir2, false); + return IndexReader.open(dir2); } }); @@ -104,8 +104,8 @@ @Override protected IndexReader openReader() throws IOException { ParallelReader pr = new ParallelReader(); - pr.add(IndexReader.open(dir1, false)); - pr.add(IndexReader.open(dir2, false)); + pr.add(IndexReader.open(dir1)); + pr.add(IndexReader.open(dir2)); return pr; } @@ -129,11 +129,11 @@ @Override protected IndexReader openReader() throws IOException { ParallelReader pr = new ParallelReader(); - pr.add(IndexReader.open(dir3, false)); - pr.add(IndexReader.open(dir4, false)); + pr.add(IndexReader.open(dir3)); + pr.add(IndexReader.open(dir4)); // Does not implement reopen, so // hits exception: - pr.add(new FilterIndexReader(IndexReader.open(dir3, false))); + pr.add(new FilterIndexReader(IndexReader.open(dir3))); return pr; } @@ -163,7 +163,7 @@ TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode( OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(newLogMergePolicy())); iwriter.commit(); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); try { int M = 3; FieldType customType = new FieldType(TextField.TYPE_STORED); @@ -200,7 +200,7 @@ } else { // recreate reader.close(); - reader = IndexReader.open(dir, false); + reader = IndexReader.open(dir); } } } finally { @@ -226,8 +226,8 @@ @Override protected IndexReader openReader() throws IOException { - return new MultiReader(IndexReader.open(dir1, false), - IndexReader.open(dir2, false)); + return new MultiReader(IndexReader.open(dir1), + IndexReader.open(dir2)); } }); @@ -251,11 +251,11 @@ @Override protected IndexReader openReader() throws IOException { - return new MultiReader(IndexReader.open(dir3, false), - IndexReader.open(dir4, false), + return new MultiReader(IndexReader.open(dir3), + IndexReader.open(dir4), // Does not implement reopen, so // hits exception: - new FilterIndexReader(IndexReader.open(dir3, false))); + new FilterIndexReader(IndexReader.open(dir3))); } }); @@ -279,20 +279,15 @@ @Override protected void modifyIndex(int i) throws IOException { - // only change norms in this index to maintain the same number of docs for each of ParallelReader's subreaders - if (i == 1) TestIndexReaderReopen.modifyIndex(i, dir1); - TestIndexReaderReopen.modifyIndex(i, dir4); TestIndexReaderReopen.modifyIndex(i, dir5); } @Override protected IndexReader openReader() throws IOException { - ParallelReader pr = new ParallelReader(); - pr.add(IndexReader.open(dir1, false)); - pr.add(IndexReader.open(dir2, false)); - MultiReader mr = new MultiReader(IndexReader.open(dir3, false), IndexReader.open(dir4, false)); - return new MultiReader(pr, mr, IndexReader.open(dir5, false)); + MultiReader mr1 = new MultiReader(IndexReader.open(dir1), IndexReader.open(dir2)); + MultiReader mr2 = new MultiReader(IndexReader.open(dir3), IndexReader.open(dir4)); + return new MultiReader(mr1, mr2, IndexReader.open(dir5)); } }); dir1.close(); @@ -347,111 +342,6 @@ assertReaderClosed(index1, true, true); assertReaderClosed(index2, true, true); } - - public void testReferenceCounting() throws IOException { - for (int mode = 0; mode < 4; mode++) { - Directory dir1 = newDirectory(); - createIndex(random, dir1, true); - - IndexReader reader0 = IndexReader.open(dir1, false); - assertRefCountEquals(1, reader0); - - assertTrue(reader0 instanceof DirectoryReader); - IndexReader[] subReaders0 = reader0.getSequentialSubReaders(); - for (int i = 0; i < subReaders0.length; i++) { - assertRefCountEquals(1, subReaders0[i]); - } - - // delete first document, so that only one of the subReaders have to be re-opened - IndexReader modifier = IndexReader.open(dir1, false); - modifier.deleteDocument(0); - modifier.close(); - - IndexReader reader1 = refreshReader(reader0, true).refreshedReader; - assertTrue(reader1 instanceof DirectoryReader); - IndexReader[] subReaders1 = reader1.getSequentialSubReaders(); - assertEquals(subReaders0.length, subReaders1.length); - - for (int i = 0; i < subReaders0.length; i++) { - if (subReaders0[i] != subReaders1[i]) { - assertRefCountEquals(1, subReaders0[i]); - assertRefCountEquals(1, subReaders1[i]); - } else { - assertRefCountEquals(2, subReaders0[i]); - } - } - - // delete first document, so that only one of the subReaders have to be re-opened - modifier = IndexReader.open(dir1, false); - modifier.deleteDocument(1); - modifier.close(); - - IndexReader reader2 = refreshReader(reader1, true).refreshedReader; - assertTrue(reader2 instanceof DirectoryReader); - IndexReader[] subReaders2 = reader2.getSequentialSubReaders(); - assertEquals(subReaders1.length, subReaders2.length); - - for (int i = 0; i < subReaders2.length; i++) { - if (subReaders2[i] == subReaders1[i]) { - if (subReaders1[i] == subReaders0[i]) { - assertRefCountEquals(3, subReaders2[i]); - } else { - assertRefCountEquals(2, subReaders2[i]); - } - } else { - assertRefCountEquals(1, subReaders2[i]); - if (subReaders0[i] == subReaders1[i]) { - assertRefCountEquals(2, subReaders2[i]); - assertRefCountEquals(2, subReaders0[i]); - } else { - assertRefCountEquals(1, subReaders0[i]); - assertRefCountEquals(1, subReaders1[i]); - } - } - } - - IndexReader reader3 = refreshReader(reader0, true).refreshedReader; - assertTrue(reader3 instanceof DirectoryReader); - IndexReader[] subReaders3 = reader3.getSequentialSubReaders(); - assertEquals(subReaders3.length, subReaders0.length); - - // try some permutations - switch (mode) { - case 0: - reader0.close(); - reader1.close(); - reader2.close(); - reader3.close(); - break; - case 1: - reader3.close(); - reader2.close(); - reader1.close(); - reader0.close(); - break; - case 2: - reader2.close(); - reader3.close(); - reader0.close(); - reader1.close(); - break; - case 3: - reader1.close(); - reader3.close(); - reader2.close(); - reader0.close(); - break; - } - - assertReaderClosed(reader0, true, true); - assertReaderClosed(reader1, true, true); - assertReaderClosed(reader2, true, true); - assertReaderClosed(reader3, true, true); - - dir1.close(); - } - } - public void testReferenceCountingMultiReader() throws IOException { for (int mode = 0; mode <=1; mode++) { @@ -460,10 +350,10 @@ Directory dir2 = newDirectory(); createIndex(random, dir2, true); - IndexReader reader1 = IndexReader.open(dir1, false); + IndexReader reader1 = IndexReader.open(dir1); assertRefCountEquals(1, reader1); - IndexReader initReader2 = IndexReader.open(dir2, false); + IndexReader initReader2 = IndexReader.open(dir2); IndexReader multiReader1 = new MultiReader(new IndexReader[] {reader1, initReader2}, (mode == 0)); modifyIndex(0, dir2); assertRefCountEquals(1 + mode, reader1); @@ -527,160 +417,6 @@ } } - - public void testReferenceCountingParallelReader() throws IOException { - for (int mode = 0; mode <=1; mode++) { - Directory dir1 = newDirectory(); - createIndex(random, dir1, false); - Directory dir2 = newDirectory(); - createIndex(random, dir2, true); - - IndexReader reader1 = IndexReader.open(dir1, false); - assertRefCountEquals(1, reader1); - - ParallelReader parallelReader1 = new ParallelReader(mode == 0); - parallelReader1.add(reader1); - IndexReader initReader2 = IndexReader.open(dir2, false); - parallelReader1.add(initReader2); - modifyIndex(1, dir2); - assertRefCountEquals(1 + mode, reader1); - - IndexReader parallelReader2 = IndexReader.openIfChanged(parallelReader1); - assertNotNull(parallelReader2); - assertNull(IndexReader.openIfChanged(parallelReader2)); - // index1 hasn't changed, so parallelReader2 should share reader1 now with multiReader1 - assertRefCountEquals(2 + mode, reader1); - - modifyIndex(0, dir1); - modifyIndex(0, dir2); - IndexReader reader2 = IndexReader.openIfChanged(reader1); - assertNotNull(reader2); - assertRefCountEquals(2 + mode, reader1); - - if (mode == 1) { - initReader2.close(); - } - - modifyIndex(4, dir1); - IndexReader reader3 = IndexReader.openIfChanged(reader2); - assertNotNull(reader3); - assertRefCountEquals(2 + mode, reader1); - assertRefCountEquals(1, reader2); - - parallelReader1.close(); - assertRefCountEquals(1 + mode, reader1); - - parallelReader1.close(); - assertRefCountEquals(1 + mode, reader1); - - if (mode == 1) { - initReader2.close(); - } - - reader1.close(); - assertRefCountEquals(1, reader1); - - parallelReader2.close(); - assertRefCountEquals(0, reader1); - - parallelReader2.close(); - assertRefCountEquals(0, reader1); - - reader3.close(); - assertRefCountEquals(0, reader1); - assertReaderClosed(reader1, true, false); - - reader2.close(); - assertRefCountEquals(0, reader1); - assertReaderClosed(reader1, true, false); - - reader2.close(); - assertRefCountEquals(0, reader1); - - reader3.close(); - assertRefCountEquals(0, reader1); - assertReaderClosed(reader1, true, true); - - dir1.close(); - dir2.close(); - } - - } - - public void testNormsRefCounting() throws IOException { - Directory dir1 = newDirectory(); - createIndex(random, dir1, false); - - IndexReader reader1 = IndexReader.open(dir1, false); - SegmentReader segmentReader1 = getOnlySegmentReader(reader1); - IndexReader modifier = IndexReader.open(dir1, false); - modifier.deleteDocument(0); - modifier.close(); - - IndexReader reader2 = IndexReader.openIfChanged(reader1); - assertNotNull(reader2); - modifier = IndexReader.open(dir1, false); - DefaultSimilarity sim = new DefaultSimilarity(); - modifier.setNorm(1, "field1", sim.encodeNormValue(50f)); - modifier.setNorm(1, "field2", sim.encodeNormValue(50f)); - modifier.close(); - - IndexReader reader3 = IndexReader.openIfChanged(reader2); - assertNotNull(reader3); - SegmentReader segmentReader3 = getOnlySegmentReader(reader3); - modifier = IndexReader.open(dir1, false); - modifier.deleteDocument(2); - modifier.close(); - - IndexReader reader4 = IndexReader.openIfChanged(reader3); - assertNotNull(reader4); - modifier = IndexReader.open(dir1, false); - modifier.deleteDocument(3); - modifier.close(); - - IndexReader reader5 = IndexReader.openIfChanged(reader3); - assertNotNull(reader5); - - // Now reader2-reader5 references reader1. reader1 and reader2 - // share the same norms. reader3, reader4, reader5 also share norms. - assertRefCountEquals(1, reader1); - assertFalse(segmentReader1.normsClosed()); - - reader1.close(); - - assertRefCountEquals(0, reader1); - assertFalse(segmentReader1.normsClosed()); - - reader2.close(); - assertRefCountEquals(0, reader1); - - // now the norms for field1 and field2 should be closed - assertTrue(segmentReader1.normsClosed("field1")); - assertTrue(segmentReader1.normsClosed("field2")); - - // but the norms for field3 and field4 should still be open - assertFalse(segmentReader1.normsClosed("field3")); - assertFalse(segmentReader1.normsClosed("field4")); - - reader3.close(); - assertRefCountEquals(0, reader1); - assertFalse(segmentReader3.normsClosed()); - reader5.close(); - assertRefCountEquals(0, reader1); - assertFalse(segmentReader3.normsClosed()); - reader4.close(); - assertRefCountEquals(0, reader1); - - // and now all norms that reader1 used should be closed - assertTrue(segmentReader1.normsClosed()); - - // now that reader3, reader4 and reader5 are closed, - // the norms that those three readers shared should be - // closed as well - assertTrue(segmentReader3.normsClosed()); - - dir1.close(); - } private void performTestsWithExceptionInReopen(TestReopen test) throws Exception { IndexReader index1 = test.openReader(); @@ -717,31 +453,20 @@ final TestReopen test = new TestReopen() { @Override protected void modifyIndex(int i) throws IOException { - if (i % 3 == 0) { - IndexReader modifier = IndexReader.open(dir, false); - DefaultSimilarity sim = new DefaultSimilarity(); - modifier.setNorm(i, "field1", sim.encodeNormValue(50f)); - modifier.close(); - } else if (i % 3 == 1) { - IndexReader modifier = IndexReader.open(dir, false); - modifier.deleteDocument(i % modifier.maxDoc()); - modifier.close(); - } else { - IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - modifier.addDocument(createDocument(n + i, 6)); - modifier.close(); - } + IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random))); + modifier.addDocument(createDocument(n + i, 6)); + modifier.close(); } @Override protected IndexReader openReader() throws IOException { - return IndexReader.open(dir, false); + return IndexReader.open(dir); } }; final List readers = Collections.synchronizedList(new ArrayList()); - IndexReader firstReader = IndexReader.open(dir, false); + IndexReader firstReader = IndexReader.open(dir); IndexReader reader = firstReader; final Random rnd = random; @@ -966,7 +691,7 @@ w.close(); - IndexReader r = IndexReader.open(dir, false); + IndexReader r = IndexReader.open(dir); if (multiSegment) { assertTrue(r.getSequentialSubReaders().length > 1); } else { @@ -1009,21 +734,12 @@ break; } case 1: { - IndexReader reader = IndexReader.open(dir, false); - DefaultSimilarity sim = new DefaultSimilarity(); - reader.setNorm(4, "field1", sim.encodeNormValue(123f)); - reader.setNorm(44, "field2", sim.encodeNormValue(222f)); - reader.setNorm(44, "field4", sim.encodeNormValue(22f)); - reader.close(); - break; - } - case 2: { IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.forceMerge(1); w.close(); break; } - case 3: { + case 2: { IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.addDocument(createDocument(101, 4)); w.forceMerge(1); @@ -1032,15 +748,7 @@ w.close(); break; } - case 4: { - IndexReader reader = IndexReader.open(dir, false); - DefaultSimilarity sim = new DefaultSimilarity(); - reader.setNorm(5, "field1", sim.encodeNormValue(123f)); - reader.setNorm(55, "field2", sim.encodeNormValue(222f)); - reader.close(); - break; - } - case 5: { + case 3: { IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.addDocument(createDocument(101, 4)); w.close(); @@ -1053,7 +761,8 @@ assertEquals(0, reader.getRefCount()); if (checkNormsClosed && reader instanceof SegmentReader) { - assertTrue(((SegmentReader) reader).normsClosed()); + // TODO: should we really assert something here? we check for open files and this is obselete... + // assertTrue(((SegmentReader) reader).normsClosed()); } if (checkSubReaders) { @@ -1103,94 +812,6 @@ protected abstract void modifyIndex(int i) throws IOException; } - public void testCloseOrig() throws Throwable { - Directory dir = newDirectory(); - createIndex(random, dir, false); - IndexReader r1 = IndexReader.open(dir, false); - IndexReader r2 = IndexReader.open(dir, false); - r2.deleteDocument(0); - r2.close(); - - IndexReader r3 = IndexReader.openIfChanged(r1); - assertNotNull(r3); - assertTrue(r1 != r3); - r1.close(); - try { - r1.document(2); - fail("did not hit exception"); - } catch (AlreadyClosedException ace) { - // expected - } - r3.close(); - dir.close(); - } - - public void testDeletes() throws Throwable { - Directory dir = newDirectory(); - createIndex(random, dir, false); // Create an index with a bunch of docs (1 segment) - - modifyIndex(0, dir); // Get delete bitVector on 1st segment - modifyIndex(5, dir); // Add a doc (2 segments) - - IndexReader r1 = IndexReader.open(dir, false); // MSR - - modifyIndex(5, dir); // Add another doc (3 segments) - - IndexReader r2 = IndexReader.openIfChanged(r1); // MSR - assertNotNull(r2); - assertNull(IndexReader.openIfChanged(r2)); - assertTrue(r1 != r2); - - SegmentReader sr1 = (SegmentReader) r1.getSequentialSubReaders()[0]; // Get SRs for the first segment from original - SegmentReader sr2 = (SegmentReader) r2.getSequentialSubReaders()[0]; // and reopened IRs - - // At this point they share the same BitVector - assertTrue(sr1.liveDocs==sr2.liveDocs); - - r2.deleteDocument(0); - - // r1 should not see the delete - final Bits r1LiveDocs = MultiFields.getLiveDocs(r1); - assertFalse(r1LiveDocs != null && !r1LiveDocs.get(0)); - - // Now r2 should have made a private copy of deleted docs: - assertTrue(sr1.liveDocs!=sr2.liveDocs); - - r1.close(); - r2.close(); - dir.close(); - } - - public void testDeletes2() throws Throwable { - Directory dir = newDirectory(); - createIndex(random, dir, false); - // Get delete bitVector - modifyIndex(0, dir); - IndexReader r1 = IndexReader.open(dir, false); - - // Add doc: - modifyIndex(5, dir); - - IndexReader r2 = IndexReader.openIfChanged(r1); - assertNotNull(r2); - assertTrue(r1 != r2); - - IndexReader[] rs2 = r2.getSequentialSubReaders(); - - SegmentReader sr1 = getOnlySegmentReader(r1); - SegmentReader sr2 = (SegmentReader) rs2[0]; - - // At this point they share the same BitVector - assertTrue(sr1.liveDocs==sr2.liveDocs); - final BitVector liveDocs = sr1.liveDocs; - r1.close(); - - r2.deleteDocument(0); - assertTrue(liveDocs==sr2.liveDocs); - r2.close(); - dir.close(); - } - private static class KeepAllCommits implements IndexDeletionPolicy { public void onInit(List commits) { } @@ -1223,7 +844,7 @@ } writer.close(); - IndexReader r = IndexReader.open(dir, false); + IndexReader r = IndexReader.open(dir); assertEquals(0, r.numDocs()); Collection commits = IndexReader.listCommits(dir); @@ -1232,14 +853,6 @@ assertNotNull(r2); assertTrue(r2 != r); - // Reader should be readOnly - try { - r2.deleteDocument(0); - fail("no exception hit"); - } catch (UnsupportedOperationException uoe) { - // expected - } - final Map s = commit.getUserData(); final int v; if (s.size() == 0) { @@ -1259,54 +872,4 @@ r.close(); dir.close(); } - - // LUCENE-1579: Make sure all SegmentReaders are new when - // reopen switches readOnly - public void testReopenChangeReadonly() throws Exception { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter( - dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setMaxBufferedDocs(-1). - setMergePolicy(newLogMergePolicy(10)) - ); - Document doc = new Document(); - doc.add(newField("number", "17", StringField.TYPE_UNSTORED)); - writer.addDocument(doc); - writer.commit(); - - // Open reader1 - IndexReader r = IndexReader.open(dir, false); - assertTrue(r instanceof DirectoryReader); - IndexReader r1 = getOnlySegmentReader(r); - final int[] ints = FieldCache.DEFAULT.getInts(r1, "number", false); - assertEquals(1, ints.length); - assertEquals(17, ints[0]); - - // Reopen to readonly w/ no chnages - IndexReader r3 = IndexReader.openIfChanged(r, true); - assertNotNull(r3); - assertTrue(((DirectoryReader) r3).readOnly); - r3.close(); - - // Add new segment - writer.addDocument(doc); - writer.commit(); - - // Reopen reader1 --> reader2 - IndexReader r2 = IndexReader.openIfChanged(r, true); - assertNotNull(r2); - r.close(); - assertTrue(((DirectoryReader) r2).readOnly); - IndexReader[] subs = r2.getSequentialSubReaders(); - final int[] ints2 = FieldCache.DEFAULT.getInts(subs[0], "number", false); - r2.close(); - - assertTrue(((SegmentReader) subs[0]).readOnly); - assertTrue(((SegmentReader) subs[1]).readOnly); - assertTrue(ints == ints2); - - writer.close(); - dir.close(); - } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java 2011-12-04 20:33:24.350258400 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java 2011-12-08 19:31:38.537246600 +0100 @@ -90,19 +90,19 @@ // add 100 documents for (i = 0; i < 100; i++) { - addDoc(writer); + addDocWithIndex(writer,i); } assertEquals(100, writer.maxDoc()); writer.close(); // delete 40 documents - reader = IndexReader.open(dir, false); + writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES)); for (i = 0; i < 40; i++) { - reader.deleteDocument(i); + writer.deleteDocuments(new Term("id", ""+i)); } - reader.close(); + writer.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(60, reader.numDocs()); reader.close(); @@ -115,7 +115,7 @@ writer.close(); // check that the index reader gives the same numbers. - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(60, reader.maxDoc()); assertEquals(60, reader.numDocs()); reader.close(); @@ -182,7 +182,7 @@ writer.close(); // now open reader: - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals("should be one document", reader.numDocs(), 1); // now open index for create: @@ -192,7 +192,7 @@ writer.close(); assertEquals("should be one document", reader.numDocs(), 1); - IndexReader reader2 = IndexReader.open(dir, true); + IndexReader reader2 = IndexReader.open(dir); assertEquals("should be one document", reader2.numDocs(), 1); reader.close(); reader2.close(); @@ -227,7 +227,7 @@ writer.commit(); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(0, reader.maxDoc()); assertEquals(0, reader.numDocs()); reader.close(); @@ -236,7 +236,7 @@ writer.commit(); writer.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(0, reader.maxDoc()); assertEquals(0, reader.numDocs()); reader.close(); @@ -258,7 +258,7 @@ } writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(100, reader.maxDoc()); assertEquals(100, reader.numDocs()); for(int j=0;j<100;j++) { @@ -452,7 +452,7 @@ } writer.close(); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(new Term("field", "aaa")), null, 1000).scoreDocs; assertEquals(300, hits.length); @@ -484,7 +484,7 @@ Term searchTerm = new Term("field", "aaa"); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals(10, hits.length); @@ -507,14 +507,14 @@ writer.addDocument(doc); } writer.close(); - reader = IndexReader.open(dir, false); + reader = IndexReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals(27, hits.length); searcher.close(); reader.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); reader.close(); dir.close(); @@ -541,7 +541,7 @@ writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(1, reader.maxDoc()); assertEquals(1, reader.numDocs()); Term t = new Term("field", "a"); @@ -586,7 +586,7 @@ } writer.close(); Term searchTerm = new Term("content", "aaa"); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("did not get right number of hits", 100, hits.length); @@ -643,7 +643,7 @@ } writer.addDocument(new Document()); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(2, reader.numDocs()); reader.close(); dir.close(); @@ -698,7 +698,6 @@ public void testVariableSchema() throws Exception { Directory dir = newDirectory(); - int delID = 0; for(int i=0;i<20;i++) { if (VERBOSE) { System.out.println("TEST: iter=" + i); @@ -730,9 +729,6 @@ writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocument(delID++); - reader.close(); if (0 == i % 4) { writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); @@ -830,7 +826,7 @@ t1.join(); // Make sure reader can read - IndexReader reader = IndexReader.open(directory, true); + IndexReader reader = IndexReader.open(directory); reader.close(); // Reopen @@ -858,7 +854,7 @@ writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); Term t = new Term("field", "x"); assertEquals(1, reader.docFreq(t)); reader.close(); @@ -885,7 +881,7 @@ doc.add(newField("", "a b c", TextField.TYPE_UNSTORED)); writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); IndexReader subreader = getOnlySegmentReader(reader); TermsEnum te = subreader.fields().terms("").iterator(null); assertEquals(new BytesRef("a"), te.next()); @@ -906,7 +902,7 @@ doc.add(newField("", "c", StringField.TYPE_UNSTORED)); writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); IndexReader subreader = getOnlySegmentReader(reader); TermsEnum te = subreader.fields().terms("").iterator(null); assertEquals(new BytesRef(""), te.next()); @@ -960,7 +956,7 @@ assertTrue(w.afterWasCalled); w.close(); - IndexReader ir = IndexReader.open(dir, true); + IndexReader ir = IndexReader.open(dir); assertEquals(0, ir.numDocs()); ir.close(); @@ -994,7 +990,7 @@ w.addDocument(doc); w.commit(); - IndexReader r = IndexReader.open(dir, false); + IndexReader r = IndexReader.open(dir); IndexSearcher s = new IndexSearcher(r); PhraseQuery pq = new PhraseQuery(); pq.add(new Term("field", "a")); @@ -1043,7 +1039,7 @@ w.addDocument(doc); w.close(); - IndexReader ir = IndexReader.open(dir, true); + IndexReader ir = IndexReader.open(dir); Document doc2 = ir.document(0); IndexableField f2 = doc2.getField("binary"); b = f2.binaryValue().bytes; @@ -1072,7 +1068,7 @@ w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); Terms tpv = r.getTermVectors(0).terms("field"); TermsEnum termsEnum = tpv.iterator(null); assertNotNull(termsEnum.next()); @@ -1136,12 +1132,12 @@ writer2.addDocument(doc); writer2.close(); - IndexReader r1 = IndexReader.open(dir2, true); + IndexReader r1 = IndexReader.open(dir2); IndexReader r2 = (IndexReader) r1.clone(); writer.addIndexes(r1, r2); writer.close(); - IndexReader r3 = IndexReader.open(dir, true); + IndexReader r3 = IndexReader.open(dir); assertEquals(5, r3.numDocs()); r3.close(); @@ -1186,7 +1182,7 @@ w.close(); w = null; _TestUtil.checkIndex(dir); - IndexReader.open(dir, true).close(); + IndexReader.open(dir).close(); // Strangely, if we interrupt a thread before // all classes are loaded, the class loader @@ -1236,7 +1232,7 @@ e.printStackTrace(System.out); } try { - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); //System.out.println("doc count=" + r.numDocs()); r.close(); } catch (Exception e) { @@ -1322,7 +1318,7 @@ w.forceMerge(1); // force segment merge. w.close(); - IndexReader ir = IndexReader.open(dir, true); + IndexReader ir = IndexReader.open(dir); Document doc2 = ir.document(0); IndexableField f3 = doc2.getField("binary"); b = f3.binaryValue().bytes; diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java 2011-11-16 23:30:28.512658100 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java 2011-12-08 19:31:32.181883100 +0100 @@ -51,21 +51,21 @@ writer.close(); Term searchTerm = new Term("content", "aaa"); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("first number of hits", 14, hits.length); searcher.close(); reader.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); for(int i=0;i<3;i++) { for(int j=0;j<11;j++) { TestIndexWriter.addDoc(writer); } - IndexReader r = IndexReader.open(dir, false); + IndexReader r = IndexReader.open(dir); searcher = new IndexSearcher(r); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("reader incorrectly sees changes from writer", 14, hits.length); @@ -78,7 +78,7 @@ writer.close(); assertFalse("reader should not be current now", reader.isCurrent()); - IndexReader r = IndexReader.open(dir, false); + IndexReader r = IndexReader.open(dir); searcher = new IndexSearcher(r); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("reader did not see changes after writer was closed", 47, hits.length); @@ -105,7 +105,7 @@ writer.close(); Term searchTerm = new Term("content", "aaa"); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("first number of hits", 14, hits.length); @@ -120,7 +120,7 @@ // Delete all docs: writer.deleteDocuments(searchTerm); - reader = IndexReader.open(dir, false); + reader = IndexReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("reader incorrectly sees changes from writer", 14, hits.length); @@ -132,7 +132,7 @@ TestIndexWriter.assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()"); - reader = IndexReader.open(dir, false); + reader = IndexReader.open(dir); searcher = new IndexSearcher(reader); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("saw changes after writer.abort", 14, hits.length); @@ -152,7 +152,7 @@ for(int j=0;j<17;j++) { TestIndexWriter.addDoc(writer); } - IndexReader r = IndexReader.open(dir, false); + IndexReader r = IndexReader.open(dir); searcher = new IndexSearcher(r); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("reader incorrectly sees changes from writer", 14, hits.length); @@ -161,7 +161,7 @@ } writer.close(); - IndexReader r = IndexReader.open(dir, false); + IndexReader r = IndexReader.open(dir); searcher = new IndexSearcher(r); hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs; assertEquals("didn't see changes after close", 218, hits.length); @@ -243,7 +243,7 @@ writer.forceMerge(1); writer.close(); - IndexReader.open(dir, true).close(); + IndexReader.open(dir).close(); long endDiskUsage = dir.getMaxUsedSizeInBytes(); @@ -287,7 +287,7 @@ writer.forceMerge(1); // Open a reader before closing (commiting) the writer: - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); // Reader should see index as multi-seg at this // point: @@ -299,7 +299,7 @@ TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after forceMerge"); // Open a reader after aborting writer: - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); // Reader should still see index as multi-segment assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().length > 1); @@ -318,7 +318,7 @@ TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after forceMerge"); // Open a reader after aborting writer: - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); // Reader should see index as one segment assertEquals("Reader incorrectly sees more than one segment", 1, reader.getSequentialSubReaders().length); @@ -398,7 +398,7 @@ for (int i = 0; i < 23; i++) TestIndexWriter.addDoc(writer); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(0, reader.numDocs()); writer.commit(); IndexReader reader2 = IndexReader.openIfChanged(reader); @@ -411,12 +411,12 @@ TestIndexWriter.addDoc(writer); assertEquals(23, reader2.numDocs()); reader2.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(23, reader.numDocs()); reader.close(); writer.commit(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(40, reader.numDocs()); reader.close(); writer.close(); @@ -473,33 +473,10 @@ assertNotNull(commit); - IndexReader r = IndexReader.open(commit, true); - assertEquals(2, r.numDocs()); - r.close(); - - // open "second", w/ writeable IndexReader & commit - r = IndexReader.open(commit, NoDeletionPolicy.INSTANCE, false); - assertEquals(2, r.numDocs()); - r.deleteDocument(0); - r.deleteDocument(1); - commitData.put("tag", "fourth"); - r.commit(commitData); - r.close(); - - // make sure "third" commit is still there - commit = null; - for(IndexCommit c : IndexReader.listCommits(dir)) { - if (c.getUserData().get("tag").equals("third")) { - commit = c; - break; - } - } - assertNotNull(commit); - dir.close(); } - public void testNoCommits() throws Exception { + public void testZeroCommits() throws Exception { // Tests that if we don't call commit(), the directory has 0 commits. This has // changed since LUCENE-2386, where before IW would always commit on a fresh // new index. @@ -532,12 +509,12 @@ for (int i = 0; i < 23; i++) TestIndexWriter.addDoc(writer); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(0, reader.numDocs()); writer.prepareCommit(); - IndexReader reader2 = IndexReader.open(dir, true); + IndexReader reader2 = IndexReader.open(dir); assertEquals(0, reader2.numDocs()); writer.commit(); @@ -555,18 +532,18 @@ assertEquals(23, reader3.numDocs()); reader3.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(23, reader.numDocs()); reader.close(); writer.prepareCommit(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(23, reader.numDocs()); reader.close(); writer.commit(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(40, reader.numDocs()); reader.close(); writer.close(); @@ -589,12 +566,12 @@ for (int i = 0; i < 23; i++) TestIndexWriter.addDoc(writer); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(0, reader.numDocs()); writer.prepareCommit(); - IndexReader reader2 = IndexReader.open(dir, true); + IndexReader reader2 = IndexReader.open(dir); assertEquals(0, reader2.numDocs()); writer.rollback(); @@ -610,18 +587,18 @@ for (int i = 0; i < 17; i++) TestIndexWriter.addDoc(writer); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(0, reader.numDocs()); reader.close(); writer.prepareCommit(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(0, reader.numDocs()); reader.close(); writer.commit(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(17, reader.numDocs()); reader.close(); writer.close(); @@ -637,7 +614,7 @@ writer.commit(); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(0, reader.numDocs()); reader.close(); dir.close(); @@ -653,7 +630,7 @@ assertEquals(0, IndexReader.getCommitUserData(dir).size()); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); // commit(Map) never called for this index assertEquals(0, r.getCommitUserData().size()); r.close(); @@ -668,7 +645,7 @@ assertEquals("test1", IndexReader.getCommitUserData(dir).get("label")); - r = IndexReader.open(dir, true); + r = IndexReader.open(dir); assertEquals("test1", r.getCommitUserData().get("label")); r.close(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java 2011-11-16 23:30:30.455769300 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java 2011-12-08 19:31:40.742372700 +0100 @@ -106,7 +106,7 @@ modifier.commit(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -114,7 +114,7 @@ modifier.commit(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(0, reader.numDocs()); reader.close(); modifier.close(); @@ -166,7 +166,7 @@ assertEquals(0, modifier.getSegmentCount()); modifier.commit(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(1, reader.numDocs()); int hitCount = getHitCount(dir, new Term("id", String.valueOf(id))); @@ -204,7 +204,7 @@ modifier.commit(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(5, reader.numDocs()); modifier.close(); reader.close(); @@ -226,7 +226,7 @@ } modifier.commit(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -236,7 +236,7 @@ modifier.commit(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(5, reader.numDocs()); reader.close(); @@ -246,7 +246,7 @@ } modifier.deleteDocuments(terms); modifier.commit(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(2, reader.numDocs()); reader.close(); @@ -269,7 +269,7 @@ } modifier.commit(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -280,7 +280,7 @@ modifier.deleteAll(); // Delete all shouldn't be on disk yet - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -292,7 +292,7 @@ modifier.commit(); // Validate there are no docs left - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(2, reader.numDocs()); reader.close(); @@ -317,7 +317,7 @@ addDoc(modifier, ++id, value); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -329,7 +329,7 @@ modifier.close(); // Validate that the docs are still there - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -372,7 +372,7 @@ modifier.close(); // Validate that the docs are still there - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); assertEquals(7, reader.numDocs()); reader.close(); @@ -567,7 +567,7 @@ // changed (transactional semantics): IndexReader newReader = null; try { - newReader = IndexReader.open(dir, true); + newReader = IndexReader.open(dir); } catch (IOException e) { e.printStackTrace(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java 2011-12-04 20:33:24.228251400 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java 2011-12-08 19:31:33.020931100 +0100 @@ -254,7 +254,7 @@ } // Confirm that when doc hits exception partway through tokenization, it's deleted: - IndexReader r2 = IndexReader.open(dir, true); + IndexReader r2 = IndexReader.open(dir); final int count = r2.docFreq(new Term("content4", "aaa")); final int count2 = r2.docFreq(new Term("content4", "ddd")); assertEquals(count, count2); @@ -300,7 +300,7 @@ } // Confirm that when doc hits exception partway through tokenization, it's deleted: - IndexReader r2 = IndexReader.open(dir, true); + IndexReader r2 = IndexReader.open(dir); final int count = r2.docFreq(new Term("content4", "aaa")); final int count2 = r2.docFreq(new Term("content4", "ddd")); assertEquals(count, count2); @@ -494,7 +494,7 @@ writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); final Term t = new Term("content", "aa"); assertEquals(3, reader.docFreq(t)); @@ -576,7 +576,7 @@ } assertTrue(hitError); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(198, reader.docFreq(new Term("content", "aa"))); reader.close(); dir.close(); @@ -631,7 +631,7 @@ if (VERBOSE) { System.out.println("TEST: open reader"); } - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); if (i == 0) { int expected = 5; assertEquals(expected, reader.docFreq(new Term("contents", "here"))); @@ -660,7 +660,7 @@ writer.forceMerge(1); writer.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); int expected = 19+(1-i)*2; assertEquals(expected, reader.docFreq(new Term("contents", "here"))); assertEquals(expected, reader.maxDoc()); @@ -746,7 +746,7 @@ writer.close(); } - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); int expected = (3+(1-i)*2)*NUM_THREAD*NUM_ITER; assertEquals("i=" + i, expected, reader.docFreq(new Term("contents", "here"))); assertEquals(expected, reader.maxDoc()); @@ -774,7 +774,7 @@ writer.forceMerge(1); writer.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); expected += 17-NUM_THREAD*NUM_ITER; assertEquals(expected, reader.docFreq(new Term("contents", "here"))); assertEquals(expected, reader.maxDoc()); @@ -845,7 +845,7 @@ failure.clearDoFail(); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(23, reader.numDocs()); reader.close(); dir.close(); @@ -1058,7 +1058,7 @@ IndexReader reader = null; try { - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); } catch (IOException e) { e.printStackTrace(System.out); fail("segmentInfos failed to retry fallback to correct segments_N file"); @@ -1105,7 +1105,7 @@ IndexReader reader = null; try { - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); fail("reader did not hit IOException on opening a corrupt index"); } catch (Exception e) { } @@ -1154,7 +1154,7 @@ IndexReader reader = null; try { - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); fail("reader did not hit IOException on opening a corrupt index"); } catch (Exception e) { } @@ -1205,7 +1205,7 @@ IndexReader reader = null; try { - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); } catch (Exception e) { fail("reader failed to open on a crashed index"); } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java 2011-11-12 00:38:46.224906200 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java 2011-12-08 19:31:39.878323300 +0100 @@ -186,7 +186,7 @@ if (0 == pass) { writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertEquals(1, reader.getSequentialSubReaders().length); reader.close(); } else { @@ -196,7 +196,7 @@ writer.addDocument(doc); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); assertTrue(reader.getSequentialSubReaders().length > 1); reader.close(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java 2011-11-10 16:27:07.933715500 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java 2011-12-08 19:31:34.168996800 +0100 @@ -192,9 +192,14 @@ } writer.close(); - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocuments(new Term("content", "aaa")); - reader.close(); + // delete some docs without merging + writer = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES) + ); + writer.deleteDocuments(new Term("content", "aaa")); + writer.close(); ldmp = new LogDocMergePolicy(); ldmp.setMergeFactor(5); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java 2011-11-18 18:58:03.207975100 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java 2011-12-08 19:31:41.557419400 +0100 @@ -78,7 +78,7 @@ private boolean verifyIndex(Directory directory, int startAt) throws IOException { boolean fail = false; - IndexReader reader = IndexReader.open(directory, true); + IndexReader reader = IndexReader.open(directory); int max = reader.maxDoc(); for (int i = 0; i < max; i++) @@ -126,8 +126,6 @@ IndexWriterConfig.DISABLE_AUTO_FLUSH)); Document document = new Document(); - document = new Document(); - FieldType customType = new FieldType(); customType.setStored(true); @@ -137,19 +135,31 @@ customType1.setStoreTermVectorPositions(true); customType1.setStoreTermVectorOffsets(true); + Field idField = newField("id", "", StringField.TYPE_UNSTORED); + document.add(idField); Field storedField = newField("stored", "stored", customType); document.add(storedField); Field termVectorField = newField("termVector", "termVector", customType1); document.add(termVectorField); - for(int i=0;i<10;i++) + for(int i=0;i<10;i++) { + idField.setValue("" + i); writer.addDocument(document); + } writer.close(); - IndexReader ir = IndexReader.open(dir, false); + IndexReader ir = IndexReader.open(dir); assertEquals(10, ir.maxDoc()); assertEquals(10, ir.numDocs()); - ir.deleteDocument(0); - ir.deleteDocument(7); + ir.close(); + + IndexWriterConfig dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMergePolicy(NoMergePolicy.COMPOUND_FILES); + writer = new IndexWriter(dir, dontMergeConfig); + writer.deleteDocuments(new Term("id", "0")); + writer.deleteDocuments(new Term("id", "7")); + writer.close(); + + ir = IndexReader.open(dir); assertEquals(8, ir.numDocs()); ir.close(); @@ -159,7 +169,7 @@ writer.forceMergeDeletes(); assertEquals(8, writer.numDocs()); writer.close(); - ir = IndexReader.open(dir, true); + ir = IndexReader.open(dir); assertEquals(8, ir.maxDoc()); assertEquals(8, ir.numDocs()); ir.close(); @@ -192,15 +202,28 @@ document.add(storedField); Field termVectorField = newField("termVector", "termVector", customType1); document.add(termVectorField); - for(int i=0;i<98;i++) + Field idField = newField("id", "", StringField.TYPE_UNSTORED); + document.add(idField); + for(int i=0;i<98;i++) { + idField.setValue("" + i); writer.addDocument(document); + } writer.close(); - IndexReader ir = IndexReader.open(dir, false); + IndexReader ir = IndexReader.open(dir); assertEquals(98, ir.maxDoc()); assertEquals(98, ir.numDocs()); - for(int i=0;i<98;i+=2) - ir.deleteDocument(i); + ir.close(); + + IndexWriterConfig dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMergePolicy(NoMergePolicy.COMPOUND_FILES); + writer = new IndexWriter(dir, dontMergeConfig); + for(int i=0;i<98;i+=2) { + writer.deleteDocuments(new Term("id", "" + i)); + } + writer.close(); + + ir = IndexReader.open(dir); assertEquals(49, ir.numDocs()); ir.close(); @@ -212,7 +235,7 @@ assertEquals(49, writer.numDocs()); writer.forceMergeDeletes(); writer.close(); - ir = IndexReader.open(dir, true); + ir = IndexReader.open(dir); assertEquals(49, ir.maxDoc()); assertEquals(49, ir.numDocs()); ir.close(); @@ -245,15 +268,27 @@ document.add(storedField); Field termVectorField = newField("termVector", "termVector", customType1); document.add(termVectorField); - for(int i=0;i<98;i++) + Field idField = newField("id", "", StringField.TYPE_UNSTORED); + document.add(idField); + for(int i=0;i<98;i++) { + idField.setValue("" + i); writer.addDocument(document); + } writer.close(); - IndexReader ir = IndexReader.open(dir, false); + IndexReader ir = IndexReader.open(dir); assertEquals(98, ir.maxDoc()); assertEquals(98, ir.numDocs()); - for(int i=0;i<98;i+=2) - ir.deleteDocument(i); + ir.close(); + + IndexWriterConfig dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMergePolicy(NoMergePolicy.COMPOUND_FILES); + writer = new IndexWriter(dir, dontMergeConfig); + for(int i=0;i<98;i+=2) { + writer.deleteDocuments(new Term("id", "" + i)); + } + writer.close(); + ir = IndexReader.open(dir); assertEquals(49, ir.numDocs()); ir.close(); @@ -264,7 +299,7 @@ ); writer.forceMergeDeletes(false); writer.close(); - ir = IndexReader.open(dir, true); + ir = IndexReader.open(dir); assertEquals(49, ir.maxDoc()); assertEquals(49, ir.numDocs()); ir.close(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java 2011-11-12 00:38:43.744501800 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java 2011-12-08 19:31:32.517902300 +0100 @@ -113,7 +113,7 @@ assertNoUnreferencedFiles(dir, "after disk full during addDocument"); // Make sure reader can open the index: - IndexReader.open(dir, true).close(); + IndexReader.open(dir).close(); } dir.close(); @@ -189,7 +189,7 @@ // Make sure starting index seems to be working properly: Term searchTerm = new Term("content", "aaa"); - IndexReader reader = IndexReader.open(startDir, true); + IndexReader reader = IndexReader.open(startDir); assertEquals("first docFreq", 57, reader.docFreq(searchTerm)); IndexSearcher searcher = newSearcher(reader); @@ -306,7 +306,7 @@ } else if (1 == method) { IndexReader readers[] = new IndexReader[dirs.length]; for(int i=0;i norms; - private ArrayList modifiedNorms; - private float lastNorm = 0; - private float normDelta = (float) 0.001; - - @Override - public void setUp() throws Exception { - super.setUp(); - similarityProviderOne = new SimilarityProviderOne(); - anlzr = new MockAnalyzer(random); - } - - /** - * Test that norms values are preserved as the index is maintained. - * Including separate norms. - * Including merging indexes with seprate norms. - * Including forceMerge. - */ - public void testNorms() throws IOException { - Directory dir1 = newDirectory(); - - norms = new ArrayList(); - modifiedNorms = new ArrayList(); - - createIndex(random, dir1); - doTestNorms(random, dir1); - - // test with a single index: index2 - ArrayList norms1 = norms; - ArrayList modifiedNorms1 = modifiedNorms; - int numDocNorms1 = numDocNorms; - - norms = new ArrayList(); - modifiedNorms = new ArrayList(); - numDocNorms = 0; - - Directory dir2 = newDirectory(); - - createIndex(random, dir2); - doTestNorms(random, dir2); - - // add index1 and index2 to a third index: index3 - Directory dir3 = newDirectory(); - - createIndex(random, dir3); - IndexWriter iw = new IndexWriter( - dir3, - newIndexWriterConfig(TEST_VERSION_CURRENT, anlzr). - setOpenMode(OpenMode.APPEND). - setMaxBufferedDocs(5). - setMergePolicy(newLogMergePolicy(3)) - ); - iw.addIndexes(dir1,dir2); - iw.forceMerge(1); - iw.close(); - - norms1.addAll(norms); - norms = norms1; - modifiedNorms1.addAll(modifiedNorms); - modifiedNorms = modifiedNorms1; - numDocNorms += numDocNorms1; - - // test with index3 - verifyIndex(dir3); - doTestNorms(random, dir3); - - // now with single segment - iw = new IndexWriter( - dir3, - newIndexWriterConfig(TEST_VERSION_CURRENT, anlzr). - setOpenMode(OpenMode.APPEND). - setMaxBufferedDocs(5). - setMergePolicy(newLogMergePolicy(3)) - ); - iw.forceMerge(1); - iw.close(); - verifyIndex(dir3); - - dir1.close(); - dir2.close(); - dir3.close(); - } - - private void doTestNorms(Random random, Directory dir) throws IOException { - int num = atLeast(1); - for (int i=0; i storedNorms = (i==1 ? modifiedNorms : norms); - DefaultSimilarity sim = (DefaultSimilarity) similarityProviderOne.get(field); - for (int j = 0; j < b.length; j++) { - float norm = sim.decodeNormValue(b[j]); - float norm1 = storedNorms.get(j).floatValue(); - assertEquals("stored norm value of "+field+" for doc "+j+" is "+norm+" - a mismatch!", norm, norm1, 0.000001); - } - } - ir.close(); - } - - private void addDocs(Random random, Directory dir, int ndocs, boolean compound) throws IOException { - IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND) - .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newLogMergePolicy())); - LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy(); - lmp.setMergeFactor(3); - lmp.setUseCompoundFile(compound); - for (int i = 0; i < ndocs; i++) { - iw.addDocument(newDoc()); - } - iw.close(); - } - - // create the next document - private Document newDoc() { - Document d = new Document(); - float boost = nextNorm("anyfield"); // in this test the same similarity is used for all fields so it does not matter what field is passed - for (int i = 0; i < 10; i++) { - Field f = newField("f"+i,"v"+i,TextField.TYPE_UNSTORED); - f.setBoost(boost); - d.add(f); - } - return d; - } - - // return unique norm values that are unchanged by encoding/decoding - private float nextNorm(String fname) { - float norm = lastNorm + normDelta; - DefaultSimilarity similarity = (DefaultSimilarity) similarityProviderOne.get(fname); - do { - float norm1 = similarity.decodeNormValue(similarity.encodeNormValue(norm)); - if (norm1 > lastNorm) { - //System.out.println(norm1+" > "+lastNorm); - norm = norm1; - break; - } - norm += normDelta; - } while (true); - norms.add(numDocNorms, Float.valueOf(norm)); - modifiedNorms.add(numDocNorms, Float.valueOf(norm)); - //System.out.println("creating norm("+numDocNorms+"): "+norm); - numDocNorms ++; - lastNorm = (norm>10 ? 0 : norm); //there's a limit to how many distinct values can be stored in a ingle byte - return norm; - } class CustomNormEncodingSimilarity extends DefaultSimilarity { @Override diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java 2011-11-12 00:38:45.741305300 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java 2011-12-08 19:31:34.628023000 +0100 @@ -66,7 +66,7 @@ // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); + SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); FieldInfos fi = reader.fieldInfos(); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms); @@ -120,7 +120,7 @@ // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); + SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); FieldInfos fi = reader.fieldInfos(); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms); @@ -168,7 +168,7 @@ // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); + SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); FieldInfos fi = reader.fieldInfos(); assertTrue("OmitNorms field bit should not be set.", !fi.fieldInfo("f1").omitNorms); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms); @@ -180,7 +180,8 @@ private void assertNoNrm(Directory dir) throws Throwable { final String[] files = dir.listAll(); for (int i = 0; i < files.length; i++) { - assertFalse(files[i].endsWith(".nrm")); + // TODO: this relies upon filenames + assertFalse(files[i].endsWith(".nrm") || files[i].endsWith(".len")); } } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java 2011-12-04 20:33:24.233251700 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java 2011-12-08 19:31:34.497015500 +0100 @@ -153,7 +153,7 @@ // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); + SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); FieldInfos fi = reader.fieldInfos(); // docs + docs = docs assertEquals(IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestOmitTf.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestOmitTf.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestOmitTf.java 2011-11-17 20:47:54.740525100 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestOmitTf.java 2011-12-08 19:31:34.013987900 +0100 @@ -100,7 +100,7 @@ // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); + SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); FieldInfos fi = reader.fieldInfos(); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions); @@ -152,7 +152,7 @@ // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); + SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); FieldInfos fi = reader.fieldInfos(); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions); @@ -195,7 +195,7 @@ // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); + SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); FieldInfos fi = reader.fieldInfos(); assertEquals("OmitTermFreqAndPositions field bit should not be set.", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestParallelReader.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestParallelReader.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestParallelReader.java 2011-11-12 00:38:46.443306500 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestParallelReader.java 2011-12-08 19:31:41.858436600 +0100 @@ -75,8 +75,8 @@ Directory dir1 = getDir1(random); Directory dir2 = getDir2(random); ParallelReader pr = new ParallelReader(); - pr.add(IndexReader.open(dir1, false)); - pr.add(IndexReader.open(dir2, false)); + pr.add(IndexReader.open(dir1)); + pr.add(IndexReader.open(dir2)); Collection fieldNames = pr.getFieldNames(IndexReader.FieldOption.ALL); assertEquals(4, fieldNames.size()); assertTrue(fieldNames.contains("f1")); @@ -102,8 +102,8 @@ w2.close(); ParallelReader pr = new ParallelReader(); - pr.add(IndexReader.open(dir1, false)); - IndexReader ir = IndexReader.open(dir2, false); + pr.add(IndexReader.open(dir1)); + IndexReader ir = IndexReader.open(dir2); try { pr.add(ir); fail("didn't get exptected exception: indexes don't have same number of documents"); @@ -115,34 +115,6 @@ dir1.close(); dir2.close(); } - - public void testIsCurrent() throws IOException { - Directory dir1 = getDir1(random); - Directory dir2 = getDir2(random); - ParallelReader pr = new ParallelReader(); - pr.add(IndexReader.open(dir1, false)); - pr.add(IndexReader.open(dir2, false)); - - assertTrue(pr.isCurrent()); - IndexReader modifier = IndexReader.open(dir1, false); - DefaultSimilarity sim = new DefaultSimilarity(); - modifier.setNorm(0, "f1", sim.encodeNormValue(100f)); - modifier.close(); - - // one of the two IndexReaders which ParallelReader is using - // is not current anymore - assertFalse(pr.isCurrent()); - - modifier = IndexReader.open(dir2, false); - modifier.setNorm(0, "f3", sim.encodeNormValue(100f)); - modifier.close(); - - // now both are not current anymore - assertFalse(pr.isCurrent()); - pr.close(); - dir1.close(); - dir2.close(); - } private void queryTest(Query query) throws IOException { ScoreDoc[] parallelHits = parallel.search(query, null, 1000).scoreDocs; @@ -177,7 +149,7 @@ w.addDocument(d2); w.close(); - IndexReader ir = IndexReader.open(dir, false); + IndexReader ir = IndexReader.open(dir); return newSearcher(ir); } @@ -186,8 +158,8 @@ dir1 = getDir1(random); dir2 = getDir2(random); ParallelReader pr = new ParallelReader(); - pr.add(IndexReader.open(dir1, false)); - pr.add(IndexReader.open(dir2, false)); + pr.add(IndexReader.open(dir1)); + pr.add(IndexReader.open(dir2)); return newSearcher(pr); } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java 2011-11-12 00:38:46.911307400 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java 2011-12-08 19:31:43.163511200 +0100 @@ -24,6 +24,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -52,8 +53,8 @@ IndexWriter iwOut = new IndexWriter(rdOut, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); ParallelReader pr = new ParallelReader(); - pr.add(IndexReader.open(rd1,true)); - pr.add(IndexReader.open(rd2,true)); + pr.add(IndexReader.open(rd1)); + pr.add(IndexReader.open(rd2)); // When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum) iwOut.addIndexes(pr); @@ -75,16 +76,27 @@ { IndexWriter iw = new IndexWriter(rd1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); Document doc = new Document(); + Field idField = newField("id", "", TextField.TYPE_UNSTORED); + doc.add(idField); FieldType customType = new FieldType(TextField.TYPE_UNSTORED); customType.setStoreTermVectors(true); doc.add(newField("test", "", customType)); + idField.setValue("1"); iw.addDocument(doc); doc.add(newField("test", "", TextField.TYPE_UNSTORED)); + idField.setValue("2"); iw.addDocument(doc); iw.close(); - IndexReader ir = IndexReader.open(rd1,false); - ir.deleteDocument(0); + IndexWriterConfig dontMergeConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setMergePolicy(NoMergePolicy.COMPOUND_FILES); + IndexWriter writer = new IndexWriter(rd1, dontMergeConfig); + + writer.deleteDocuments(new Term("id", "1")); + writer.close(); + IndexReader ir = IndexReader.open(rd1); + assertEquals(2, ir.maxDoc()); + assertEquals(1, ir.numDocs()); ir.close(); iw = new IndexWriter(rd1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); @@ -104,8 +116,8 @@ IndexWriter iwOut = new IndexWriter(rdOut, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); ParallelReader pr = new ParallelReader(); - pr.add(IndexReader.open(rd1,true)); - pr.add(IndexReader.open(rd2,true)); + pr.add(IndexReader.open(rd1)); + pr.add(IndexReader.open(rd2)); // When unpatched, Lucene crashes here with an ArrayIndexOutOfBoundsException (caused by TermVectorsWriter) iwOut.addIndexes(pr); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java 2011-12-04 20:33:24.292255100 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java 2011-12-08 19:31:36.925154400 +0100 @@ -58,8 +58,8 @@ iw2.close(); - this.ir1 = IndexReader.open(rd1, true); - this.ir2 = IndexReader.open(rd2, true); + this.ir1 = IndexReader.open(rd1); + this.ir2 = IndexReader.open(rd2); } @Override diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestPayloads.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestPayloads.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestPayloads.java 2011-11-16 23:30:28.840676900 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestPayloads.java 2011-12-08 19:31:35.679083100 +0100 @@ -111,7 +111,7 @@ // flush writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram, false)); + SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); FieldInfos fi = reader.fieldInfos(); assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads); assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads); @@ -138,7 +138,7 @@ // flush writer.close(); - reader = getOnlySegmentReader(IndexReader.open(ram, false)); + reader = getOnlySegmentReader(IndexReader.open(ram)); fi = reader.fieldInfos(); assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads); assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads); @@ -213,7 +213,7 @@ * Verify the index * first we test if all payloads are stored correctly */ - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); byte[] verifyPayloadData = new byte[payloadDataLength]; offset = 0; @@ -326,7 +326,7 @@ // flush writer.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); tp = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), fieldName, @@ -526,7 +526,7 @@ ingesters[i].join(); } writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); TermsEnum terms = MultiFields.getFields(reader).terms(field).iterator(null); Bits liveDocs = MultiFields.getLiveDocs(reader); DocsAndPositionsEnum tp = null; diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestRollback.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestRollback.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestRollback.java 2011-10-21 19:50:39.948951500 +0200 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestRollback.java 2011-12-08 19:31:37.786203600 +0100 @@ -48,7 +48,7 @@ } w.rollback(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); assertEquals("index should contain same number of docs post rollback", 5, r.numDocs()); r.close(); dir.close(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java 2011-12-04 20:33:24.253252900 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java 2011-12-08 19:31:35.360064900 +0100 @@ -54,8 +54,8 @@ SegmentInfo info1 = DocHelper.writeDoc(random, merge1Dir, doc1); DocHelper.setupDoc(doc2); SegmentInfo info2 = DocHelper.writeDoc(random, merge2Dir, doc2); - reader1 = SegmentReader.get(true, info1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); - reader2 = SegmentReader.get(true, info2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + reader1 = SegmentReader.get(info1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + reader2 = SegmentReader.get(info2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); } @Override @@ -86,7 +86,7 @@ assertTrue(docsMerged == 2); final FieldInfos fieldInfos = mergeState.fieldInfos; //Should be able to open a new SegmentReader against the new directory - SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, + SegmentReader mergedReader = SegmentReader.getRW(new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, codec, fieldInfos), true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); assertTrue(mergedReader != null); @@ -158,27 +158,6 @@ } assertFalse("should not have been able to create a .cfs with .del and .s* files", doFail); - // Create an index w/ .s* - w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)); - doc = new Document(); - doc.add(new TextField("c", "test")); - w.addDocument(doc); - w.close(); - IndexReader r = IndexReader.open(dir, false); - r.setNorm(0, "c", (byte) 1); - r.close(); - - // Assert that SM fails if .s* exists - SegmentInfos sis = new SegmentInfos(); - sis.read(dir); - try { - IndexWriter.createCompoundFile(dir, "b2", MergeState.CheckAbort.NONE, sis.info(0), newIOContext(random)); - doFail = true; // should never get here - } catch (AssertionError e) { - // expected - } - assertFalse("should not have been able to create a .cfs with .del and .s* files", doFail); - dir.close(); } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java 2011-12-04 20:33:24.464264900 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java 2011-12-08 19:31:43.025503300 +0100 @@ -41,7 +41,7 @@ dir = newDirectory(); DocHelper.setupDoc(testDoc); SegmentInfo info = DocHelper.writeDoc(random, dir, testDoc); - reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.READ); + reader = SegmentReader.get(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.READ); } @Override @@ -77,7 +77,7 @@ Document docToDelete = new Document(); DocHelper.setupDoc(docToDelete); SegmentInfo info = DocHelper.writeDoc(random, dir, docToDelete); - SegmentReader deleteReader = SegmentReader.get(false, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + SegmentReader deleteReader = SegmentReader.getRW(info, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); assertTrue(deleteReader != null); assertTrue(deleteReader.numDocs() == 1); deleteReader.deleteDocument(0); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java 2011-12-04 20:33:24.222251100 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java 2011-12-08 19:31:32.693912400 +0100 @@ -56,7 +56,7 @@ public void testTermDocs(int indexDivisor) throws IOException { //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(true, info, indexDivisor, newIOContext(random)); + SegmentReader reader = SegmentReader.get(info, indexDivisor, newIOContext(random)); assertTrue(reader != null); assertEquals(indexDivisor, reader.getTermInfosIndexDivisor()); @@ -79,7 +79,7 @@ public void testBadSeek(int indexDivisor) throws IOException { { //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(true, info, indexDivisor, newIOContext(random)); + SegmentReader reader = SegmentReader.get(info, indexDivisor, newIOContext(random)); assertTrue(reader != null); DocsEnum termDocs = _TestUtil.docs(random, reader, "textField2", @@ -93,7 +93,7 @@ } { //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(true, info, indexDivisor, newIOContext(random)); + SegmentReader reader = SegmentReader.get(info, indexDivisor, newIOContext(random)); assertTrue(reader != null); DocsEnum termDocs = _TestUtil.docs(random, reader, "junk", @@ -130,7 +130,7 @@ writer.forceMerge(1); writer.close(); - IndexReader reader = IndexReader.open(dir, null, true, indexDivisor); + IndexReader reader = IndexReader.open(dir, indexDivisor); DocsEnum tdocs = _TestUtil.docs(random, reader, ta.field(), diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java 2011-11-16 23:30:28.522658700 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java 2011-12-08 19:31:32.328891500 +0100 @@ -78,7 +78,7 @@ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); addDoc(writer, "aaa bbb"); writer.close(); - SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir, false)); + SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir)); TermsEnum terms = reader.fields().terms("content").iterator(null); assertNotNull(terms.next()); assertEquals("aaa", terms.term().utf8ToString()); @@ -102,7 +102,7 @@ private void verifyDocFreq() throws IOException { - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); TermsEnum termEnum = MultiFields.getTerms(reader, "content").iterator(null); // create enumeration of all terms diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java 2011-11-16 23:30:29.841734100 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java 2011-12-08 19:31:37.108164900 +0100 @@ -20,6 +20,8 @@ import java.io.IOException; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; @@ -27,8 +29,15 @@ public class TestSizeBoundedForceMerge extends LuceneTestCase { private void addDocs(IndexWriter writer, int numDocs) throws IOException { + addDocs(writer, numDocs, false); + } + + private void addDocs(IndexWriter writer, int numDocs, boolean withID) throws IOException { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); + if (withID) { + doc.add(new Field("id", "" + i, StringField.TYPE_UNSTORED)); + } writer.addDocument(doc); } writer.commit(); @@ -286,12 +295,9 @@ addDocs(writer, 5); addDocs(writer, 3); - writer.close(); - // delete the last document, so that the last segment is merged. - IndexReader r = IndexReader.open(dir, false); - r.deleteDocument(r.numDocs() - 1); - r.close(); + writer.deleteDocuments(new Term("id", "10")); + writer.close(); conf = newWriterConfig(); LogMergePolicy lmp = new LogDocMergePolicy(); @@ -315,7 +321,7 @@ IndexWriterConfig conf = newWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); - addDocs(writer, 3); + addDocs(writer, 3, true); writer.close(); @@ -340,14 +346,12 @@ IndexWriterConfig conf = newWriterConfig(); IndexWriter writer = new IndexWriter(dir, conf); - addDocs(writer, 5); + addDocs(writer, 5, true); - writer.close(); - // delete the last document - IndexReader r = IndexReader.open(dir, false); - r.deleteDocument(r.numDocs() - 1); - r.close(); + + writer.deleteDocuments(new Term("id", "4")); + writer.close(); conf = newWriterConfig(); LogMergePolicy lmp = new LogDocMergePolicy(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java 2011-10-21 19:50:38.713880800 +0200 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java 2011-12-08 19:31:31.815862200 +0100 @@ -54,7 +54,7 @@ } protected void checkMaxDoc(IndexCommit commit, int expectedMaxDoc) throws Exception { - IndexReader reader = IndexReader.open(commit, true); + IndexReader reader = IndexReader.open(commit); try { assertEquals(expectedMaxDoc, reader.maxDoc()); } finally { @@ -245,7 +245,7 @@ assertSnapshotExists(dir, sdp, numSnapshots); // open a reader on a snapshot - should succeed. - IndexReader.open(sdp.getSnapshot("snapshot0"), true).close(); + IndexReader.open(sdp.getSnapshot("snapshot0")).close(); // open a new IndexWriter w/ no snapshots to keep and assert that all snapshots are gone. sdp = getDeletionPolicy(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestStressIndexing.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestStressIndexing.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestStressIndexing.java 2011-11-16 23:30:30.576776200 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestStressIndexing.java 2011-12-08 19:31:41.256402100 +0100 @@ -103,7 +103,7 @@ @Override public void doWork() throws Throwable { for (int i=0; i<100; i++) { - IndexReader ir = IndexReader.open(directory, true); + IndexReader ir = IndexReader.open(directory); IndexSearcher is = new IndexSearcher(ir); is.close(); ir.close(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java 2011-12-04 20:33:24.332257400 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java 2011-12-08 19:31:38.173225800 +0100 @@ -272,8 +272,8 @@ } public static void verifyEquals(Directory dir1, Directory dir2, String idField) throws Throwable { - IndexReader r1 = IndexReader.open(dir1, true); - IndexReader r2 = IndexReader.open(dir2, true); + IndexReader r1 = IndexReader.open(dir1); + IndexReader r2 = IndexReader.open(dir2); verifyEquals(r1, r2, idField); r1.close(); r2.close(); diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSumDocFreq.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSumDocFreq.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestSumDocFreq.java 2011-11-16 23:30:30.287759700 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestSumDocFreq.java 2011-12-08 19:31:40.253344800 +0100 @@ -17,9 +17,9 @@ * limitations under the License. */ -import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -38,11 +38,14 @@ RandomIndexWriter writer = new RandomIndexWriter(random, dir); Document doc = new Document(); + Field id = newField("id", "", StringField.TYPE_UNSTORED); Field field1 = newField("foo", "", TextField.TYPE_UNSTORED); Field field2 = newField("bar", "", TextField.TYPE_UNSTORED); + doc.add(id); doc.add(field1); doc.add(field2); for (int i = 0; i < numDocs; i++) { + id.setValue("" + i); char ch1 = (char) _TestUtil.nextInt(random, 'a', 'z'); char ch2 = (char) _TestUtil.nextInt(random, 'a', 'z'); field1.setValue("" + ch1 + " " + ch2); @@ -53,26 +56,20 @@ } IndexReader ir = writer.getReader(); - writer.close(); assertSumDocFreq(ir); ir.close(); - ir = IndexReader.open(dir, false); int numDeletions = atLeast(20); for (int i = 0; i < numDeletions; i++) { - ir.deleteDocument(random.nextInt(ir.maxDoc())); + writer.deleteDocuments(new Term("id", "" + random.nextInt(numDocs))); } - ir.close(); - - IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - w.forceMerge(1); - w.close(); + writer.forceMerge(1); + writer.close(); - ir = IndexReader.open(dir, true); + ir = IndexReader.open(dir); assertSumDocFreq(ir); ir.close(); - dir.close(); } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java 2011-11-16 23:30:30.086748200 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java 2011-12-08 19:31:38.871265700 +0100 @@ -59,7 +59,7 @@ w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); Terms vector = r.getTermVectors(0).terms("field"); assertNotNull(vector); TermsEnum termsEnum = vector.iterator(null); @@ -119,7 +119,7 @@ w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); @@ -156,7 +156,7 @@ w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); @@ -197,7 +197,7 @@ w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); @@ -235,7 +235,7 @@ w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); @@ -274,7 +274,7 @@ w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); @@ -325,7 +325,7 @@ w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); @@ -372,7 +372,7 @@ w.addDocument(doc); w.close(); - IndexReader r = IndexReader.open(dir, true); + IndexReader r = IndexReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); @@ -433,7 +433,7 @@ writer.forceMerge(1); writer.close(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); for(int i=0;i lastScore); - lastScore = scores[i]; - } - store.close(); - } -} diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java trunk-lusolr3/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java 2011-10-21 19:50:37.719824000 +0200 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java 2011-12-08 19:31:27.983643000 +0100 @@ -491,7 +491,7 @@ writer.close(); // Get searcher - final IndexReader reader = IndexReader.open(dir, true); + final IndexReader reader = IndexReader.open(dir); final IndexSearcher searcher = newSearcher(reader); // Control (make sure docs indexed) diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java trunk-lusolr3/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java 2011-10-21 19:50:37.750825800 +0200 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java 2011-12-08 19:31:28.189654800 +0100 @@ -76,7 +76,7 @@ * @throws Exception */ public void testVerifyIndex() throws Exception { - final IndexReader reader = IndexReader.open(mDirectory, true); + final IndexReader reader = IndexReader.open(mDirectory); assertEquals(8, reader.numDocs()); reader.close(); } diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java trunk-lusolr3/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java 2011-11-04 19:04:21.564358100 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java 2011-12-08 19:31:43.970557400 +0100 @@ -256,25 +256,33 @@ doc.add(newField("id", "" + i, TextField.TYPE_STORED)); writer.addDocument(doc); } - writer.close(); dir.allIndexInputs.clear(); - IndexReader reader = IndexReader.open(dir, false); + IndexReader reader = IndexReader.open(writer, true); Term aaa = new Term("content", "aaa"); Term bbb = new Term("content", "bbb"); - Term ccc = new Term("content", "ccc"); - assertEquals(37, reader.docFreq(ccc)); - reader.deleteDocument(0); - assertEquals(37, reader.docFreq(aaa)); - dir.tweakBufferSizes(); - reader.deleteDocument(4); - assertEquals(reader.docFreq(bbb), 37); + + reader.close(); + dir.tweakBufferSizes(); - + writer.deleteDocuments(new Term("id", "0")); + reader = IndexReader.open(writer, true); IndexSearcher searcher = newSearcher(reader); ScoreDoc[] hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs; dir.tweakBufferSizes(); + assertEquals(36, hits.length); + + reader.close(); + searcher.close(); + + dir.tweakBufferSizes(); + writer.deleteDocuments(new Term("id", "4")); + reader = IndexReader.open(writer, true); + searcher = newSearcher(reader); + + hits = searcher.search(new TermQuery(bbb), null, 1000).scoreDocs; + dir.tweakBufferSizes(); assertEquals(35, hits.length); dir.tweakBufferSizes(); hits = searcher.search(new TermQuery(new Term("id", "33")), null, 1000).scoreDocs; @@ -283,6 +291,7 @@ hits = searcher.search(new TermQuery(aaa), null, 1000).scoreDocs; dir.tweakBufferSizes(); assertEquals(35, hits.length); + writer.close(); searcher.close(); reader.close(); } finally { diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java trunk-lusolr3/lucene/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java 2011-12-03 13:28:27.357127500 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java 2011-12-08 19:31:43.818548700 +0100 @@ -94,7 +94,7 @@ public void testNoDir() throws Throwable { Directory dir = newFSSwitchDirectory(Collections.emptySet()); try { - IndexReader.open(dir, true); + IndexReader.open(dir); fail("did not hit expected exception"); } catch (NoSuchDirectoryException nsde) { // expected diff -urN -x .svn -x site trunk-lusolr4/lucene/src/test/org/apache/lucene/store/TestLockFactory.java trunk-lusolr3/lucene/src/test/org/apache/lucene/store/TestLockFactory.java --- trunk-lusolr4/lucene/src/test/org/apache/lucene/store/TestLockFactory.java 2011-11-16 23:30:32.069861600 +0100 +++ trunk-lusolr3/lucene/src/test/org/apache/lucene/store/TestLockFactory.java 2011-12-08 19:31:43.344521600 +0100 @@ -345,7 +345,7 @@ Query query = new TermQuery(new Term("content", "aaa")); for(int i=0;i : 1000 - CommitIndex(original) - CloseIndex - } - - OpenReader(false,original) - DeleteByPercent(5) - { "SearchSameRdr5" Search > : 500 - FlushReader(5%) - CloseReader - PrintReader(5%) - - OpenReader(false,5%) - DeleteByPercent(10) - { "SearchSameRdr10" Search > : 500 - FlushReader(10%) - CloseReader - PrintReader(10%) - - OpenReader(false,10%) - DeleteByPercent(20) - { "SearchSameRdr20" Search > : 500 - FlushReader(20%) - CloseReader - PrintReader(20%) - - OpenReader(false,20%) - DeleteByPercent(60) - { "SearchSameRdr60" Search > : 500 - FlushReader(60%) - CloseReader - PrintReader(60%) - - OpenReader(false,60%) - DeleteByPercent(75) - { "SearchSameRdr75" Search > : 500 - FlushReader(75%) - CloseReader - PrintReader(75%) - - # Test lower percentage of deletes (so undeleteAll is used) - OpenReader(false,75%) - DeleteByPercent(7) - { "SearchSameRdr7" Search > : 500 - FlushReader(7%) - CloseReader - PrintReader(7%) - - NewRound - -} : 1 - -RepSumByName -RepSumByPrefRound MAddDocs diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/conf/deletes.alg trunk-lusolr3/modules/benchmark/conf/deletes.alg --- trunk-lusolr4/modules/benchmark/conf/deletes.alg 2011-11-12 00:38:41.092497100 +0100 +++ trunk-lusolr3/modules/benchmark/conf/deletes.alg 1970-01-01 01:00:00.000000000 +0100 @@ -1,70 +0,0 @@ -#/** -# * Licensed to the Apache Software Foundation (ASF) under one or more -# * contributor license agreements. See the NOTICE file distributed with -# * this work for additional information regarding copyright ownership. -# * The ASF licenses this file to You under the Apache License, Version 2.0 -# * (the "License"); you may not use this file except in compliance with -# * the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -------------------------------------------------------- -# Deletes: what is the cost of deleting documents? -# -------------------------------------------------------- - -# ------------------------------------------------------------------------------------- -# multi val params are iterated by NewRound's, added to reports, start with column name. - -merge.factor=mrg:10 -max.buffered=buf:100 -compound=true - -analyzer=org.apache.lucene.analysis.standard.StandardAnalyzer -directory=FSDirectory -#directory=RamDirectory - -doc.stored=true -doc.tokenized=true -doc.term.vector=false -log.step=10000 -log.step.DeleteDoc=100 - -docs.dir=reuters-out -#docs.dir=reuters-111 - -content.source=org.apache.lucene.benchmark.byTask.feeds.SingleDocSource -#content.source=org.apache.lucene.benchmark.byTask.feeds.ReutersContentSource - -query.maker=org.apache.lucene.benchmark.byTask.feeds.SimpleQueryMaker -#query.maker=org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker - -# task at this depth or less would print when they start -task.max.depth.log=1 - -log.queries=false -# ------------------------------------------------------------------------------------- - -ResetSystemErase - -CreateIndex -CloseIndex - -{ "Populate" - OpenIndex - { AddDoc(10) > : 200000 - ForcMerge(1) - CloseIndex -> - -{ "Deletions" - OpenReader(false) DeleteDoc CloseReader -} : 4000 - -RepSumByName - diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java --- trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java 2011-10-19 14:21:21.809401100 +0200 +++ trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java 2011-12-08 19:31:16.068961500 +0100 @@ -21,7 +21,6 @@ import org.apache.lucene.benchmark.byTask.PerfRunData; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexReader; /** * Commits the IndexWriter. @@ -51,14 +50,6 @@ IndexWriter iw = getRunData().getIndexWriter(); if (iw != null) { iw.commit(commitUserData); - } else { - IndexReader r = getRunData().getIndexReader(); - if (r != null) { - r.commit(commitUserData); - r.decRef(); - } else { - throw new IllegalStateException("neither IndexWriter nor IndexReader is currently open"); - } } return 1; diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteByPercentTask.java trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteByPercentTask.java --- trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteByPercentTask.java 2011-10-19 14:21:21.935408300 +0200 +++ trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteByPercentTask.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,95 +0,0 @@ -package org.apache.lucene.benchmark.byTask.tasks; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Random; - -import org.apache.lucene.benchmark.byTask.PerfRunData; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.util.Bits; - -/** - * Deletes a percentage of documents from an index randomly - * over the number of documents. The parameter, X, is in - * percent. EG 50 means 1/2 of all documents will be - * deleted. - * - *

NOTE: the param is an absolute percentage of - * maxDoc(). This means if you delete 50%, and then delete - * 50% again, the 2nd delete will do nothing. - * - *

Parameters: - *

    - *
  • delete.percent.rand.seed - defines the seed to - * initialize Random (default 1717) - *
- */ -public class DeleteByPercentTask extends PerfTask { - double percent; - int numDeleted = 0; - final Random random; - - public DeleteByPercentTask(PerfRunData runData) { - super(runData); - random = new Random(runData.getConfig().get("delete.percent.rand.seed", 1717)); - } - - @Override - public void setParams(String params) { - super.setParams(params); - percent = Double.parseDouble(params)/100; - } - - @Override - public boolean supportsParams() { - return true; - } - - @Override - public int doLogic() throws Exception { - IndexReader r = getRunData().getIndexReader(); - int maxDoc = r.maxDoc(); - int numDeleted = 0; - // percent is an absolute target: - int numToDelete = ((int) (maxDoc * percent)) - r.numDeletedDocs(); - if (numToDelete < 0) { - r.undeleteAll(); - numToDelete = (int) (maxDoc * percent); - } - while (numDeleted < numToDelete) { - double delRate = ((double) (numToDelete-numDeleted))/r.numDocs(); - Bits liveDocs = MultiFields.getLiveDocs(r); - int doc = 0; - while (doc < maxDoc && numDeleted < numToDelete) { - if ((liveDocs == null || liveDocs.get(doc)) && random.nextDouble() <= delRate) { - r.deleteDocument(doc); - numDeleted++; - if (liveDocs == null) { - liveDocs = MultiFields.getLiveDocs(r); - assert liveDocs != null; - } - } - doc++; - } - } - System.out.println("--> processed (delete) " + numDeleted + " docs"); - r.decRef(); - return numDeleted; - } -} diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java --- trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java 2011-10-19 14:21:21.974410500 +0200 +++ trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,90 +0,0 @@ -package org.apache.lucene.benchmark.byTask.tasks; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.benchmark.byTask.PerfRunData; -import org.apache.lucene.index.IndexReader; - -/** - * Delete a document by docid. If no docid param is supplied, deletes doc with - * id = last-deleted-doc + doc.delete.step. - */ -public class DeleteDocTask extends PerfTask { - - /** - * Gap between ids of deleted docs, applies when no docid param is provided. - */ - public static final int DEFAULT_DOC_DELETE_STEP = 8; - - public DeleteDocTask(PerfRunData runData) { - super(runData); - } - - private int deleteStep = -1; - private static int lastDeleted = -1; - - private int docid = -1; - private boolean byStep = true; - - @Override - public int doLogic() throws Exception { - IndexReader r = getRunData().getIndexReader(); - r.deleteDocument(docid); - lastDeleted = docid; - r.decRef(); - return 1; // one work item done here - } - - /* (non-Javadoc) - * @see org.apache.lucene.benchmark.byTask.tasks.PerfTask#setup() - */ - @Override - public void setup() throws Exception { - super.setup(); - if (deleteStep<0) { - deleteStep = getRunData().getConfig().get("doc.delete.step",DEFAULT_DOC_DELETE_STEP); - } - // set the docid to be deleted - docid = (byStep ? lastDeleted + deleteStep : docid); - } - - @Override - protected String getLogMessage(int recsCount) { - return "deleted " + recsCount + " docs, last deleted: " + lastDeleted; - } - - /** - * Set the params (docid only) - * @param params docid to delete, or -1 for deleting by delete gap settings. - */ - @Override - public void setParams(String params) { - super.setParams(params); - docid = (int) Float.parseFloat(params); - byStep = (docid < 0); - } - - /* (non-Javadoc) - * @see org.apache.lucene.benchmark.byTask.tasks.PerfTask#supportsParams() - */ - @Override - public boolean supportsParams() { - return true; - } - -} diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/FlushReaderTask.java trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/FlushReaderTask.java --- trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/FlushReaderTask.java 2011-10-19 14:21:22.000412000 +0200 +++ trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/FlushReaderTask.java 1970-01-01 01:00:00.000000000 +0100 @@ -1,58 +0,0 @@ -package org.apache.lucene.benchmark.byTask.tasks; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import org.apache.lucene.benchmark.byTask.PerfRunData; -import org.apache.lucene.index.IndexReader; - -public class FlushReaderTask extends PerfTask { - String userData = null; - - public FlushReaderTask(PerfRunData runData) { - super(runData); - } - - @Override - public boolean supportsParams() { - return true; - } - - @Override - public void setParams(String params) { - super.setParams(params); - userData = params; - } - - @Override - public int doLogic() throws IOException { - IndexReader reader = getRunData().getIndexReader(); - if (userData != null) { - Map map = new HashMap(); - map.put(OpenReaderTask.USER_DATA, userData); - reader.flush(map); - } else { - reader.flush(); - } - reader.decRef(); - return 1; - } -} diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenReaderTask.java trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenReaderTask.java --- trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenReaderTask.java 2011-10-19 14:21:21.958409600 +0200 +++ trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenReaderTask.java 2011-12-08 19:31:16.406980800 +0100 @@ -22,20 +22,17 @@ import java.util.Map; import org.apache.lucene.benchmark.byTask.PerfRunData; -import org.apache.lucene.benchmark.byTask.utils.Config; import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexReader; import org.apache.lucene.store.Directory; /** * Open an index reader. *
Other side effects: index reader object in perfRunData is set. - *
Optional params readOnly,commitUserData eg. OpenReader(false,commit1) + *
Optional params commitUserData eg. OpenReader(false,commit1) */ public class OpenReaderTask extends PerfTask { public static final String USER_DATA = "userData"; - private boolean readOnly = true; private String commitUserData = null; public OpenReaderTask(PerfRunData runData) { @@ -45,22 +42,11 @@ @Override public int doLogic() throws IOException { Directory dir = getRunData().getDirectory(); - Config config = getRunData().getConfig(); IndexReader r = null; - final IndexDeletionPolicy deletionPolicy; - if (readOnly) { - deletionPolicy = null; - } else { - deletionPolicy = CreateIndexTask.getIndexDeletionPolicy(config); - } if (commitUserData != null) { - r = IndexReader.open(OpenReaderTask.findIndexCommit(dir, commitUserData), - deletionPolicy, - readOnly); + r = IndexReader.open(OpenReaderTask.findIndexCommit(dir, commitUserData)); } else { - r = IndexReader.open(dir, - deletionPolicy, - readOnly); + r = IndexReader.open(dir); } getRunData().setIndexReader(r); // We transfer reference to the run data @@ -74,10 +60,7 @@ if (params != null) { String[] split = params.split(","); if (split.length > 0) { - readOnly = Boolean.valueOf(split[0]).booleanValue(); - } - if (split.length > 1) { - commitUserData = split[1]; + commitUserData = split[0]; } } } diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PerfTask.java trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PerfTask.java --- trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PerfTask.java 2011-10-21 19:50:32.519526500 +0200 +++ trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PerfTask.java 2011-12-08 19:31:16.283973800 +0100 @@ -36,7 +36,7 @@ * logging. *
  • log.step.[class Task Name] - specifies the same as 'log.step', only for a * particular task name. For example, log.step.AddDoc will be applied only for - * {@link AddDocTask}, but not for {@link DeleteDocTask}. It's a way to control + * {@link AddDocTask}. It's a way to control * per task logging settings. If you want to omit logging for any other task, * include log.step=-1. The syntax is "log.step." together with the Task's * 'short' name (i.e., without the 'Task' part). diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PrintReaderTask.java trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PrintReaderTask.java --- trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PrintReaderTask.java 2011-10-19 14:21:21.806400900 +0200 +++ trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/PrintReaderTask.java 2011-12-08 19:31:15.909952400 +0100 @@ -44,11 +44,9 @@ Directory dir = getRunData().getDirectory(); IndexReader r = null; if (userData == null) - r = IndexReader.open(dir, true); + r = IndexReader.open(dir); else - r = IndexReader.open(OpenReaderTask.findIndexCommit(dir, userData), - null, - true); + r = IndexReader.open(OpenReaderTask.findIndexCommit(dir, userData)); System.out.println("--> numDocs:"+r.numDocs()+" dels:"+r.numDeletedDocs()); r.close(); return 1; diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java --- trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java 2011-11-12 22:14:08.731988800 +0100 +++ trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java 2011-12-08 19:31:16.534988100 +0100 @@ -84,7 +84,7 @@ if (searcher == null) { // open our own reader Directory dir = getRunData().getDirectory(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); searcher = new IndexSearcher(reader); closeSearcher = true; } else { diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/QualityQueriesFinder.java trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/QualityQueriesFinder.java --- trunk-lusolr4/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/QualityQueriesFinder.java 2011-11-16 23:30:12.642750400 +0100 +++ trunk-lusolr3/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/QualityQueriesFinder.java 2011-12-08 19:31:14.875893300 +0100 @@ -86,7 +86,7 @@ private String [] bestTerms(String field,int numTerms) throws IOException { PriorityQueue pq = new TermsDfQueue(numTerms); - IndexReader ir = IndexReader.open(dir, true); + IndexReader ir = IndexReader.open(dir); try { int threshold = ir.maxDoc() / 10; // ignore words too common. Terms terms = MultiFields.getTerms(ir, field); diff -urN -x .svn -x site trunk-lusolr4/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java trunk-lusolr3/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java --- trunk-lusolr4/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java 2011-12-04 20:33:22.577157000 +0100 +++ trunk-lusolr3/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java 2011-12-08 19:31:14.222855900 +0100 @@ -102,7 +102,7 @@ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) .setOpenMode(OpenMode.APPEND)); iw.close(); - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); assertEquals("1000 docs were added to the index, this is what we expect to find!",1000,ir.numDocs()); ir.close(); } @@ -167,7 +167,7 @@ "{ AddDoc } : 100", "ForceMerge(1)", "CloseIndex", - "OpenReader(true)", + "OpenReader", "{ CountingHighlighterTest(size[1],highlight[1],mergeContiguous[true],maxFrags[1],fields[body]) } : 200", "CloseReader", }; @@ -188,7 +188,7 @@ // now we should be able to open the index for write. IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(), new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); iw.close(); - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); assertEquals("100 docs were added to the index, this is what we expect to find!",100,ir.numDocs()); ir.close(); } @@ -206,7 +206,7 @@ "{ AddDoc } : 1000", "ForceMerge(1)", "CloseIndex", - "OpenReader(false)", + "OpenReader", "{ CountingHighlighterTest(size[1],highlight[1],mergeContiguous[true],maxFrags[1],fields[body]) } : 200", "CloseReader", }; @@ -227,7 +227,7 @@ // now we should be able to open the index for write. IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(), new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); iw.close(); - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); assertEquals("1000 docs were added to the index, this is what we expect to find!",1000,ir.numDocs()); ir.close(); } @@ -300,7 +300,7 @@ // now we should be able to open the index for write. IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(), new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND)); iw.close(); - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); assertEquals("1 docs were added to the index, this is what we expect to find!",1,ir.numDocs()); ir.close(); } @@ -331,7 +331,7 @@ // 3. execute the algorithm (required in every "logic" test) Benchmark benchmark = execBenchmark(algLines); - IndexReader r = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader r = IndexReader.open(benchmark.getRunData().getDirectory()); DocTermsIndex idx = FieldCache.DEFAULT.getTermsIndex(r, "country"); final int maxDoc = r.maxDoc(); assertEquals(1000, maxDoc); @@ -367,7 +367,7 @@ Benchmark benchmark = execBenchmark(algLines); // 3. test number of docs in the index - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); int ndocsExpected = 20; // first 20 reuters docs. assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs()); ir.close(); @@ -432,7 +432,7 @@ .setOpenMode(OpenMode.APPEND)); iw.close(); - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); assertEquals(numLines + " lines were created but " + ir.numDocs() + " docs are in the index", numLines, ir.numDocs()); ir.close(); @@ -476,7 +476,7 @@ } // Separately count how many tokens are actually in the index: - IndexReader reader = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader reader = IndexReader.open(benchmark.getRunData().getDirectory()); assertEquals(NUM_DOCS, reader.numDocs()); int totalTokenCount2 = 0; @@ -535,7 +535,7 @@ Benchmark benchmark = execBenchmark(algLines); // 3. test number of docs in the index - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); int ndocsExpected = 2 * 20; // first 20 reuters docs. assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs()); ir.close(); @@ -572,7 +572,7 @@ Benchmark benchmark = execBenchmark(algLines); // 3. test number of docs in the index - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); int ndocsExpected = 20; // first 20 reuters docs. assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs()); ir.close(); @@ -609,7 +609,7 @@ Benchmark benchmark = execBenchmark(algLines); // 3. test number of docs in the index - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); int ndocsExpected = 20; // first 20 reuters docs. assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs()); ir.close(); @@ -623,40 +623,6 @@ } } - public void testDeleteByPercent() throws Exception { - // 1. alg definition (required in every "logic" test) - String algLines[] = { - "# ----- properties ", - "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource", - "docs.file=" + getReuters20LinesFile(), - "ram.flush.mb=-1", - "max.buffered=2", - "content.source.log.step=3", - "doc.term.vector=false", - "content.source.forever=false", - "directory=RAMDirectory", - "doc.stored=false", - "doc.tokenized=false", - "debug.level=1", - "# ----- alg ", - "CreateIndex", - "{ \"AddDocs\" AddDoc > : * ", - "CloseIndex()", - "OpenReader(false)", - "DeleteByPercent(20)", - "CloseReader" - }; - - // 2. execute the algorithm (required in every "logic" test) - Benchmark benchmark = execBenchmark(algLines); - - // 3. test number of docs in the index - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); - int ndocsExpected = 16; // first 20 reuters docs, minus 20% - assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs()); - ir.close(); - } - /** * Test that we can set merge scheduler". */ @@ -690,7 +656,7 @@ benchmark.getRunData().getIndexWriter().close(); // 3. test number of docs in the index - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); int ndocsExpected = 20; // first 20 reuters docs. assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs()); ir.close(); @@ -736,7 +702,7 @@ benchmark.getRunData().getIndexWriter().close(); // 3. test number of docs in the index - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); int ndocsExpected = 20; // first 20 reuters docs. assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs()); ir.close(); @@ -780,7 +746,7 @@ assertFalse(((LogMergePolicy) writer.getConfig().getMergePolicy()).getUseCompoundFile()); writer.close(); Directory dir = benchmark.getRunData().getDirectory(); - IndexReader reader = IndexReader.open(dir, true); + IndexReader reader = IndexReader.open(dir); Fields tfv = reader.getTermVectors(0); assertNotNull(tfv); assertTrue(tfv.getUniqueFieldCount() > 0); @@ -856,7 +822,7 @@ Benchmark benchmark = execBenchmark(algLines); // 3. test number of docs in the index - IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory(), true); + IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory()); int ndocsExpected = 20; // first 20 reuters docs. assertEquals("wrong number of docs in the index!", ndocsExpected, ir.numDocs()); ir.close(); diff -urN -x .svn -x site trunk-lusolr4/modules/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleMain.java trunk-lusolr3/modules/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleMain.java --- trunk-lusolr4/modules/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleMain.java 2011-11-03 20:10:13.338256000 +0100 +++ trunk-lusolr3/modules/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleMain.java 2011-12-08 19:31:10.918666900 +0100 @@ -57,7 +57,7 @@ // open readers TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); - IndexReader indexReader = IndexReader.open(indexDir, true); + IndexReader indexReader = IndexReader.open(indexDir); ExampleUtils.log("search the sample documents..."); List facetRes = SimpleSearcher.searchWithFacets(indexReader, taxo); @@ -82,7 +82,7 @@ // open readers TaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir); - IndexReader indexReader = IndexReader.open(indexDir, true); + IndexReader indexReader = IndexReader.open(indexDir); ExampleUtils.log("search the sample documents..."); List facetRes = SimpleSearcher.searchWithDrillDown(indexReader, taxo); diff -urN -x .svn -x site trunk-lusolr4/modules/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java trunk-lusolr3/modules/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java --- trunk-lusolr4/modules/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java 2011-11-03 20:10:12.807855100 +0100 +++ trunk-lusolr3/modules/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java 2011-12-08 19:31:09.066561000 +0100 @@ -323,19 +323,6 @@ assertTrue("Should be obtained from cache at 8th attempt",totalCounts == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null)); - // delete a doc from the reader and commit - should recompute - origReader.close(); - origReader = readers[0].indexReader; - readers[0].indexReader = IndexReader.open(origReader.directory(),false); - initCache(); - totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null); - prevGen = assertRecomputed(totalCounts, prevGen, "after opening a writable reader - 9th attempt!"); - // now do the delete - readers[0].indexReader.deleteDocument(1); - readers[0].indexReader.commit(null); - totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null); - prevGen = assertRecomputed(totalCounts, prevGen, "after deleting docs the index - 10th attempt!"); - origReader.close(); readers[0].close(); r2.close(); diff -urN -x .svn -x site trunk-lusolr4/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestIndexClose.java trunk-lusolr3/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestIndexClose.java --- trunk-lusolr4/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestIndexClose.java 2011-11-29 08:51:27.445569500 +0100 +++ trunk-lusolr3/modules/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestIndexClose.java 2011-12-08 19:31:09.788602300 +0100 @@ -147,7 +147,7 @@ } @Override protected IndexReader openIndexReader(Directory dir) throws CorruptIndexException, IOException { - return new InstrumentedIndexReader(IndexReader.open(dir,true)); + return new InstrumentedIndexReader(IndexReader.open(dir)); } } diff -urN -x .svn -x site trunk-lusolr4/modules/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java trunk-lusolr3/modules/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java --- trunk-lusolr4/modules/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java 2011-10-21 19:49:58.456578300 +0200 +++ trunk-lusolr3/modules/facet/src/test/org/apache/lucene/facet/util/TestScoredDocIDsUtils.java 2011-12-08 19:31:09.587590800 +0100 @@ -231,14 +231,11 @@ for (int docNum = 0; docNum < nDocs; docNum++) { writer.addDocument(docFactory.getDoc(docNum)); } - writer.close(); - // Delete documents marked for deletion - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocuments(new Term(DocumentFactory.field, DocumentFactory.delTxt)); - reader.close(); + writer.deleteDocuments(new Term(DocumentFactory.field, DocumentFactory.delTxt)); + writer.close(); // Open a fresh read-only reader with the deletions in place - return IndexReader.open(dir, true); + return IndexReader.open(dir); } } diff -urN -x .svn -x site trunk-lusolr4/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java trunk-lusolr3/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java --- trunk-lusolr4/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java 2011-10-21 19:49:59.576642300 +0200 +++ trunk-lusolr3/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java 2011-12-08 19:31:11.697711500 +0100 @@ -76,7 +76,7 @@ } d.close(); writer.close(); - reader = IndexReader.open(dir, true); + reader = IndexReader.open(dir); searcher = newSearcher(reader); } diff -urN -x .svn -x site trunk-lusolr4/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java trunk-lusolr3/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java --- trunk-lusolr4/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java 2011-10-25 14:13:12.459543300 +0200 +++ trunk-lusolr3/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java 2011-12-08 19:31:11.508700700 +0100 @@ -63,7 +63,7 @@ IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null)); writer.commit(); try { - IndexReader reader = new SlowMultiReaderWrapper(IndexReader.open(ramDir, true)); + IndexReader reader = new SlowMultiReaderWrapper(IndexReader.open(ramDir)); try { assertNull(filter.getDocIdSet((AtomicReaderContext) reader.getTopReaderContext(), reader.getLiveDocs())); } diff -urN -x .svn -x site trunk-lusolr4/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java trunk-lusolr3/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java --- trunk-lusolr4/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java 2011-11-12 00:38:39.392094200 +0100 +++ trunk-lusolr3/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java 2011-12-08 19:31:07.761486300 +0100 @@ -84,7 +84,7 @@ public void testFieldNonExistent() throws IOException { try { - indexReader = IndexReader.open(store, true); + indexReader = IndexReader.open(store); ld = new LuceneDictionary(indexReader, "nonexistent_field"); it = ld.getWordsIterator(); @@ -98,7 +98,7 @@ public void testFieldAaa() throws IOException { try { - indexReader = IndexReader.open(store, true); + indexReader = IndexReader.open(store); ld = new LuceneDictionary(indexReader, "aaa"); it = ld.getWordsIterator(); @@ -114,7 +114,7 @@ public void testFieldContents_1() throws IOException { try { - indexReader = IndexReader.open(store, true); + indexReader = IndexReader.open(store); ld = new LuceneDictionary(indexReader, "contents"); it = ld.getWordsIterator(); @@ -144,7 +144,7 @@ public void testFieldContents_2() throws IOException { try { - indexReader = IndexReader.open(store, true); + indexReader = IndexReader.open(store); ld = new LuceneDictionary(indexReader, "contents"); it = ld.getWordsIterator(); @@ -176,7 +176,7 @@ public void testFieldZzz() throws IOException { try { - indexReader = IndexReader.open(store, true); + indexReader = IndexReader.open(store); ld = new LuceneDictionary(indexReader, "zzz"); it = ld.getWordsIterator(); @@ -194,7 +194,7 @@ public void testSpellchecker() throws IOException { Directory dir = newDirectory(); SpellChecker sc = new SpellChecker(dir); - indexReader = IndexReader.open(store, true); + indexReader = IndexReader.open(store); sc.indexDictionary(new LuceneDictionary(indexReader, "contents"), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false); String[] suggestions = sc.suggestSimilar("Tam", 1); assertEquals(1, suggestions.length); diff -urN -x .svn -x site trunk-lusolr4/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java trunk-lusolr3/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java --- trunk-lusolr4/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java 2011-11-04 17:00:13.478352000 +0100 +++ trunk-lusolr3/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java 2011-12-08 19:31:07.946496900 +0100 @@ -104,7 +104,7 @@ public void testBuild() throws CorruptIndexException, IOException { - IndexReader r = IndexReader.open(userindex, true); + IndexReader r = IndexReader.open(userindex); spellChecker.clearIndex(); @@ -144,7 +144,7 @@ } public void testComparator() throws Exception { - IndexReader r = IndexReader.open(userindex, true); + IndexReader r = IndexReader.open(userindex); Directory compIdx = newDirectory(); SpellChecker compareSP = new SpellCheckerMock(compIdx, new LevensteinDistance(), new SuggestWordFrequencyComparator()); addwords(r, compareSP, "field3"); @@ -162,7 +162,7 @@ } public void testBogusField() throws Exception { - IndexReader r = IndexReader.open(userindex, true); + IndexReader r = IndexReader.open(userindex); Directory compIdx = newDirectory(); SpellChecker compareSP = new SpellCheckerMock(compIdx, new LevensteinDistance(), new SuggestWordFrequencyComparator()); addwords(r, compareSP, "field3"); @@ -177,7 +177,7 @@ } public void testSuggestModes() throws Exception { - IndexReader r = IndexReader.open(userindex, true); + IndexReader r = IndexReader.open(userindex); spellChecker.clearIndex(); addwords(r, spellChecker, "field1"); @@ -337,7 +337,7 @@ } private int numdoc() throws IOException { - IndexReader rs = IndexReader.open(spellindex, true); + IndexReader rs = IndexReader.open(spellindex); int num = rs.numDocs(); assertTrue(num != 0); //System.out.println("num docs: " + num); @@ -346,7 +346,7 @@ } public void testClose() throws IOException { - IndexReader r = IndexReader.open(userindex, true); + IndexReader r = IndexReader.open(userindex); spellChecker.clearIndex(); String field = "field1"; addwords(r, spellChecker, "field1"); @@ -402,7 +402,7 @@ */ public void testConcurrentAccess() throws IOException, InterruptedException { assertEquals(1, searchers.size()); - final IndexReader r = IndexReader.open(userindex, true); + final IndexReader r = IndexReader.open(userindex); spellChecker.clearIndex(); assertEquals(2, searchers.size()); addwords(r, spellChecker, "field1"); diff -urN -x .svn -x site trunk-lusolr4/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java trunk-lusolr3/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java --- trunk-lusolr4/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java 2011-10-19 14:18:03.972085400 +0200 +++ trunk-lusolr3/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java 2011-12-08 19:31:06.902437200 +0100 @@ -90,7 +90,7 @@ @Test @SuppressWarnings("unchecked") - public void testCompositePk_FullImportNoCommit() throws Exception { + public void testCompositePk_FullImportWithoutCommit() throws Exception { List parentRow = new ArrayList(); parentRow.add(createMap("id", "10")); MockDataSource.setIterator("select * from x", parentRow.iterator()); diff -urN -x .svn -x site trunk-lusolr4/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport.properties trunk-lusolr3/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport.properties --- trunk-lusolr4/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport.properties 2011-10-24 23:05:42.996747900 +0200 +++ trunk-lusolr3/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport.properties 2011-10-08 20:54:45.614491500 +0200 @@ -1,4 +1,4 @@ -#Mon Oct 24 17:05:42 EDT 2011 -x.last_index_time=2011-10-24 17\:05\:42 -parent.last_index_time=2011-10-24 17\:05\:41 -last_index_time=2011-10-24 17\:05\:42 +#Sun Oct 09 02:54:45 CST 2011 +x.last_index_time=2011-10-09 07\:54\:45 +parent.last_index_time=2011-10-09 07\:54\:44 +last_index_time=2011-10-09 07\:54\:45 diff -urN -x .svn -x site trunk-lusolr4/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport.properties trunk-lusolr3/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport.properties --- trunk-lusolr4/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport.properties 2011-10-24 23:05:25.368716900 +0200 +++ trunk-lusolr3/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport.properties 2011-10-08 20:54:20.138060900 +0200 @@ -1,4 +1,12 @@ -#Tue Oct 25 06:05:25 JST 2011 -52610164385987.last_index_time=2011-10-25 06\:05\:20 -52614007685693.last_index_time=2011-10-25 06\:05\:24 -last_index_time=2011-10-25 06\:05\:24 +#Sun Oct 09 08:39:20 CHADT 2011 +671731620306014.last_index_time=2011-10-08 04\:10\:34 +614318696771652.last_index_time=2011-10-07 22\:14\:29 +914610009025367.last_index_time=2011-07-24 05\:58\:26 +367136114823460.last_index_time=2011-10-04 23\:07\:27 +706752385985115.last_index_time=2011-10-09 08\:39\:15 +914607944999274.last_index_time=2011-07-24 05\:58\:24 +671728676611752.last_index_time=2011-10-08 04\:10\:31 +last_index_time=2011-10-09 08\:39\:19 +367133405711046.last_index_time=2011-10-04 23\:07\:24 +706756185399115.last_index_time=2011-10-09 08\:39\:19 +614315276961683.last_index_time=2011-10-07 22\:14\:25 diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java trunk-lusolr3/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java --- trunk-lusolr4/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java 2011-11-04 17:00:12.325286000 +0100 +++ trunk-lusolr3/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java 2011-12-08 19:31:04.449296900 +0100 @@ -57,10 +57,9 @@ * Creates a new IndexReader instance using the given Directory. * * @param indexDir indexDir index location - * @param readOnly return readOnly IndexReader * @return An IndexReader instance * @throws IOException */ - public abstract IndexReader newReader(Directory indexDir, boolean readOnly) + public abstract IndexReader newReader(Directory indexDir) throws IOException; } diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/java/org/apache/solr/core/SolrCore.java trunk-lusolr3/solr/core/src/java/org/apache/solr/core/SolrCore.java --- trunk-lusolr4/solr/core/src/java/org/apache/solr/core/SolrCore.java 2011-12-08 17:54:53.617224200 +0100 +++ trunk-lusolr3/solr/core/src/java/org/apache/solr/core/SolrCore.java 2011-12-08 19:31:04.619306600 +0100 @@ -314,12 +314,7 @@ // gets a non-caching searcher public SolrIndexSearcher newSearcher(String name) throws IOException { - return newSearcher(name, false); - } - - // gets a non-caching searcher - public SolrIndexSearcher newSearcher(String name, boolean readOnly) throws IOException { - return new SolrIndexSearcher(this, getNewIndexDir(), schema, getSolrConfig().mainIndexConfig, name, readOnly, false, directoryFactory); + return new SolrIndexSearcher(this, getNewIndexDir(), schema, getSolrConfig().mainIndexConfig, name, false, directoryFactory); } @@ -1149,7 +1144,7 @@ } else { // verbose("non-reopen START:"); - tmp = new SolrIndexSearcher(this, newIndexDir, schema, getSolrConfig().mainIndexConfig, "main", true, true, directoryFactory); + tmp = new SolrIndexSearcher(this, newIndexDir, schema, getSolrConfig().mainIndexConfig, "main", true, directoryFactory); // verbose("non-reopen DONE: searcher=",tmp); } } catch (Throwable th) { diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java trunk-lusolr3/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java --- trunk-lusolr4/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java 2011-11-04 17:00:12.251281800 +0100 +++ trunk-lusolr3/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java 2011-12-08 19:31:04.292287900 +0100 @@ -29,12 +29,8 @@ */ public class StandardIndexReaderFactory extends IndexReaderFactory { - /* (non-Javadoc) - * @see org.apache.solr.core.IndexReaderFactory#newReader(org.apache.lucene.store.Directory, boolean) - */ @Override - public IndexReader newReader(Directory indexDir, boolean readOnly) - throws IOException { - return IndexReader.open(indexDir, null, readOnly, termInfosIndexDivisor); + public IndexReader newReader(Directory indexDir) throws IOException { + return IndexReader.open(indexDir, termInfosIndexDivisor); } } diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java trunk-lusolr3/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java --- trunk-lusolr4/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java 2011-11-04 17:00:12.570300000 +0100 +++ trunk-lusolr3/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java 2011-12-08 19:31:05.386350500 +0100 @@ -214,7 +214,7 @@ Directory dir = dirFactory.get(dirNames[i], core.getSolrConfig().mainIndexConfig.lockType); dirsToBeReleased[i] = dir; // TODO: why doesn't this use the IR factory? what is going on here? - readersToBeClosed[i] = IndexReader.open(dir, true); + readersToBeClosed[i] = IndexReader.open(dir); } } diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java trunk-lusolr3/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java --- trunk-lusolr4/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java 2011-12-04 20:33:20.755052800 +0100 +++ trunk-lusolr3/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java 2011-12-08 19:31:04.048274000 +0100 @@ -107,9 +107,9 @@ private Collection storedHighlightFieldNames; private DirectoryFactory directoryFactory; - public SolrIndexSearcher(SolrCore core, String path, IndexSchema schema, SolrIndexConfig config, String name, boolean readOnly, boolean enableCache, DirectoryFactory directoryFactory) throws IOException { + public SolrIndexSearcher(SolrCore core, String path, IndexSchema schema, SolrIndexConfig config, String name, boolean enableCache, DirectoryFactory directoryFactory) throws IOException { // we don't need to reserve the directory because we get it from the factory - this(core, schema,name, core.getIndexReaderFactory().newReader(directoryFactory.get(path, config.lockType), readOnly), true, enableCache, false, directoryFactory); + this(core, schema,name, core.getIndexReaderFactory().newReader(directoryFactory.get(path, config.lockType)), true, enableCache, false, directoryFactory); } public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, IndexReader r, boolean closeReader, boolean enableCache, boolean reserveDirectory, DirectoryFactory directoryFactory) { diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java trunk-lusolr3/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java --- trunk-lusolr4/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java 2011-11-12 00:38:39.329694100 +0100 +++ trunk-lusolr3/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java 2011-12-08 19:31:05.728370000 +0100 @@ -111,7 +111,7 @@ writer.forceMerge(1); writer.close(); - dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir, true), + dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir), WORD_FIELD_NAME, 0.0f); } else { // check if character encoding is defined diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java trunk-lusolr3/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java --- trunk-lusolr4/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java 2011-11-04 17:00:12.725308900 +0100 +++ trunk-lusolr3/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java 2011-12-08 19:31:05.869378100 +0100 @@ -64,7 +64,7 @@ if (sourceLocation != null) { try { FSDirectory luceneIndexDir = FSDirectory.open(new File(sourceLocation)); - this.reader = IndexReader.open(luceneIndexDir, true); + this.reader = IndexReader.open(luceneIndexDir); } catch (IOException e) { throw new RuntimeException(e); } diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java trunk-lusolr3/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java --- trunk-lusolr4/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java 2011-10-21 19:47:13.337134000 +0200 +++ trunk-lusolr3/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java 2011-12-08 19:30:58.165937500 +0100 @@ -61,10 +61,9 @@ static volatile boolean newReaderCalled = false; @Override - public IndexReader newReader(Directory indexDir, boolean readOnly) - throws IOException { + public IndexReader newReader(Directory indexDir) throws IOException { TestIndexReaderFactory.newReaderCalled = true; - return IndexReader.open(indexDir, readOnly); + return IndexReader.open(indexDir); } } diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/test/org/apache/solr/core/TestQuerySenderListener.java trunk-lusolr3/solr/core/src/test/org/apache/solr/core/TestQuerySenderListener.java --- trunk-lusolr4/solr/core/src/test/org/apache/solr/core/TestQuerySenderListener.java 2011-10-21 19:47:13.391137100 +0200 +++ trunk-lusolr3/solr/core/src/test/org/apache/solr/core/TestQuerySenderListener.java 2011-12-08 19:30:58.307945600 +0100 @@ -75,7 +75,7 @@ assertNotNull("Event is null", evt); assertTrue(evt + " is not equal to " + EventParams.FIRST_SEARCHER, evt.equals(EventParams.FIRST_SEARCHER) == true); - SolrIndexSearcher newSearcher = new SolrIndexSearcher(core, core.getNewIndexDir(), core.getSchema(), core.getSolrConfig().mainIndexConfig, "testQuerySenderListener", true, false, core.getDirectoryFactory()); + SolrIndexSearcher newSearcher = new SolrIndexSearcher(core, core.getNewIndexDir(), core.getSchema(), core.getSolrConfig().mainIndexConfig, "testQuerySenderListener", false, core.getDirectoryFactory()); qsl.newSearcher(newSearcher, currentSearcher); evt = mock.req.getParams().get(EventParams.EVENT); diff -urN -x .svn -x site trunk-lusolr4/solr/core/src/test/org/apache/solr/core/TestQuerySenderNoQuery.java trunk-lusolr3/solr/core/src/test/org/apache/solr/core/TestQuerySenderNoQuery.java --- trunk-lusolr4/solr/core/src/test/org/apache/solr/core/TestQuerySenderNoQuery.java 2011-10-21 19:47:13.424138900 +0200 +++ trunk-lusolr3/solr/core/src/test/org/apache/solr/core/TestQuerySenderNoQuery.java 2011-12-08 19:30:58.448953700 +0100 @@ -75,7 +75,7 @@ assertNotNull("Mock is null", mock); assertNull("Req (firstsearcher) is not null", mock.req); - SolrIndexSearcher newSearcher = new SolrIndexSearcher(core, core.getNewIndexDir(), core.getSchema(), core.getSolrConfig().mainIndexConfig, "testQuerySenderNoQuery", true, false, core.getDirectoryFactory()); + SolrIndexSearcher newSearcher = new SolrIndexSearcher(core, core.getNewIndexDir(), core.getSchema(), core.getSolrConfig().mainIndexConfig, "testQuerySenderNoQuery", false, core.getDirectoryFactory()); qsl.newSearcher(newSearcher, currentSearcher); // get newSearcher. assertNull("Req (newsearcher) is not null", mock.req); Files trunk-lusolr4/solr/example/exampledocs/post.jar and trunk-lusolr3/solr/example/exampledocs/post.jar differ