Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 768911) +++ CHANGES.txt (working copy) @@ -50,6 +50,13 @@ ... + 3. LUCENE-1604: IndexReader.norms(String field) is now allowed to + return null if the field has no norms, as long as you've + previously called IndexReader.setDisableFakeNorms(true). This + setting now defaults to false (to preserve the fake norms back + compatible behavior) but in 3.0 will be hardwired to true. (Shon + Vella via Mike McCandless). + API Changes 1. LUCENE-1419: Add expert API to set custom indexing chain. This API is Index: src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexReader.java (revision 768911) +++ src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -1377,10 +1377,17 @@ String curField = (String) it1.next(); byte[] norms1 = index1.norms(curField); byte[] norms2 = index2.norms(curField); - assertEquals(norms1.length, norms2.length); - for (int i = 0; i < norms1.length; i++) { - assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", norms1[i], norms2[i]); - } + if (norms1 != null && norms2 != null) + { + assertEquals(norms1.length, norms2.length); + for (int i = 0; i < norms1.length; i++) { + assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", norms1[i], norms2[i]); + } + } + else + { + assertSame(norms1, norms2); + } } // check deletions Index: src/test/org/apache/lucene/index/TestSegmentReader.java =================================================================== --- src/test/org/apache/lucene/index/TestSegmentReader.java (revision 768911) +++ src/test/org/apache/lucene/index/TestSegmentReader.java (working copy) @@ -29,8 +29,6 @@ import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.MockRAMDirectory; -import org.apache.lucene.analysis.WhitespaceAnalyzer; -import org.apache.lucene.search.Similarity; public class TestSegmentReader extends LuceneTestCase { private RAMDirectory dir = new RAMDirectory(); @@ -173,16 +171,21 @@ assertEquals(reader.hasNorms(f.name()), !f.getOmitNorms()); assertEquals(reader.hasNorms(f.name()), !DocHelper.noNorms.containsKey(f.name())); if (!reader.hasNorms(f.name())) { - // test for fake norms of 1.0 + // test for fake norms of 1.0 or null depending on the flag byte [] norms = reader.norms(f.name()); - assertEquals(norms.length,reader.maxDoc()); - for (int j=0; jNOTE: If this field does not store norms, then + * this method call will silently do nothing. + * * @see #norms(String) * @see Similarity#decodeNorm(byte) * @throws StaleReaderException if the index has changed @@ -1275,4 +1280,26 @@ public long getUniqueTermCount() throws IOException { throw new UnsupportedOperationException("this reader does not implement getUniqueTermCount()"); } + + /** Expert: Return the state of the flag that disables fakes norms in favor of representing the absence of field norms with null. + * @return true if fake norms are disabled + * @deprecated This currently defaults to false (to remain + * back-compatible), but in 3.0 it will be hardwired to + * true, meaning the norms() methods will return null for + * fields that had disabled norms. + */ + public boolean getDisableFakeNorms() { + return disableFakeNorms; + } + + /** Expert: Set the state of the flag that disables fakes norms in favor of representing the absence of field norms with null. + * @param disableFakeNorms true to disable fake norms, false to preserve the legacy behavior + * @deprecated This currently defaults to false (to remain + * back-compatible), but in 3.0 it will be hardwired to + * true, meaning the norms() methods will return null for + * fields that had disabled norms. + */ + public void setDisableFakeNorms(boolean disableFakeNorms) { + this.disableFakeNorms = disableFakeNorms; + } } Index: src/java/org/apache/lucene/index/MultiSegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/MultiSegmentReader.java (revision 768911) +++ src/java/org/apache/lucene/index/MultiSegmentReader.java (working copy) @@ -18,6 +18,7 @@ */ import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -27,6 +28,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.store.Directory; /** @@ -262,15 +264,18 @@ } protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { - if (infos.size() == 1) { + DirectoryIndexReader reader; + if (infos.size() == 1) { // The index has only one segment now, so we can't refresh the MultiSegmentReader. // Return a new [ReadOnly]SegmentReader instead - return SegmentReader.get(openReadOnly, infos, infos.info(0), false); + reader = SegmentReader.get(openReadOnly, infos, infos.info(0), false); } else if (openReadOnly) { - return new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, doClone); + reader = new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, doClone); } else { - return new MultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, false, doClone); - } + reader = new MultiSegmentReader(directory, infos, closeDirectory, subReaders, starts, normsCache, false, doClone); + } + reader.setDisableFakeNorms(getDisableFakeNorms()); + return reader; } public TermFreqVector[] getTermFreqVectors(int n) throws IOException { @@ -397,7 +402,7 @@ if (bytes != null) return bytes; // cache hit if (!hasNorms(field)) - return fakeNorms(); + return getDisableFakeNorms() ? null : fakeNorms(); bytes = new byte[maxDoc()]; for (int i = 0; i < subReaders.length; i++) @@ -410,12 +415,13 @@ throws IOException { ensureOpen(); byte[] bytes = (byte[])normsCache.get(field); - if (bytes==null && !hasNorms(field)) bytes=fakeNorms(); - if (bytes != null) // cache hit + if (bytes==null && !hasNorms(field)) + Arrays.fill(result, offset, result.length, DefaultSimilarity.encodeNorm(1.0f)); + else if (bytes != null) // cache hit System.arraycopy(bytes, 0, result, offset, maxDoc()); - - for (int i = 0; i < subReaders.length; i++) // read from segments - subReaders[i].norms(field, result, offset + starts[i]); + else + for (int i = 0; i < subReaders.length; i++) // read from segments + subReaders[i].norms(field, result, offset + starts[i]); } protected void doSetNorm(int n, String field, byte value) @@ -514,6 +520,12 @@ throw new IllegalStateException("no readers"); } + public void setDisableFakeNorms(boolean disableFakeNorms) { + super.setDisableFakeNorms(disableFakeNorms); + for (int i = 0; i < subReaders.length; i++) + subReaders[i].setDisableFakeNorms(disableFakeNorms); + } + static class MultiTermEnum extends TermEnum { private SegmentMergeQueue queue; Index: contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java =================================================================== --- contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java (revision 768911) +++ contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java (working copy) @@ -72,13 +72,15 @@ private void testNorms(IndexReader r) throws IOException { byte[] norms; norms = r.norms("foo"); - assertNotNull(norms); - assertEquals(0, norms.length); - norms = new byte[10]; - Arrays.fill(norms, (byte)10); - r.norms("foo", norms, 10); - for (byte b : norms) { - assertEquals((byte)10, b); + if (!r.getDisableFakeNorms()) { + assertNotNull(norms); + assertEquals(0, norms.length); + norms = new byte[10]; + Arrays.fill(norms, (byte)10); + r.norms("foo", norms, 10); + for (byte b : norms) { + assertEquals((byte)10, b); + } } } Index: contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java =================================================================== --- contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (revision 768911) +++ contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (working copy) @@ -247,39 +247,41 @@ byte[] aprioriNorms = aprioriReader.norms((String) field); byte[] testNorms = testReader.norms((String) field); - assertEquals(aprioriNorms.length, testNorms.length); + if (!aprioriReader.getDisableFakeNorms()) { + assertEquals(aprioriNorms.length, testNorms.length); - for (int i = 0; i < aprioriNorms.length; i++) { - assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i], testNorms[i]); - } + for (int i = 0; i < aprioriNorms.length; i++) { + assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i], testNorms[i]); + } - // test norms as used by multireader + // test norms as used by multireader - aprioriNorms = new byte[aprioriReader.maxDoc()]; - aprioriReader.norms((String) field, aprioriNorms, 0); + aprioriNorms = new byte[aprioriReader.maxDoc()]; + aprioriReader.norms((String) field, aprioriNorms, 0); - testNorms = new byte[testReader.maxDoc()]; - testReader.norms((String) field, testNorms, 0); + testNorms = new byte[testReader.maxDoc()]; + testReader.norms((String) field, testNorms, 0); - assertEquals(aprioriNorms.length, testNorms.length); + assertEquals(aprioriNorms.length, testNorms.length); - for (int i = 0; i < aprioriNorms.length; i++) { - assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i], testNorms[i]); - } + for (int i = 0; i < aprioriNorms.length; i++) { + assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i], testNorms[i]); + } - // test norms as used by multireader + // test norms as used by multireader - aprioriNorms = new byte[aprioriReader.maxDoc() + 10]; - aprioriReader.norms((String) field, aprioriNorms, 10); + aprioriNorms = new byte[aprioriReader.maxDoc() + 10]; + aprioriReader.norms((String) field, aprioriNorms, 10); - testNorms = new byte[testReader.maxDoc() + 10]; - testReader.norms((String) field, testNorms, 10); + testNorms = new byte[testReader.maxDoc() + 10]; + testReader.norms((String) field, testNorms, 10); - assertEquals(aprioriNorms.length, testNorms.length); - - for (int i = 0; i < aprioriNorms.length; i++) { - assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i], testNorms[i]); + assertEquals(aprioriNorms.length, testNorms.length); + + for (int i = 0; i < aprioriNorms.length; i++) { + assertEquals("norms does not equals for field " + field + " in document " + i, aprioriNorms[i], testNorms[i]); + } } } Index: contrib/miscellaneous/src/test/org/apache/lucene/index/TestFieldNormModifier.java =================================================================== --- contrib/miscellaneous/src/test/org/apache/lucene/index/TestFieldNormModifier.java (revision 768911) +++ contrib/miscellaneous/src/test/org/apache/lucene/index/TestFieldNormModifier.java (working copy) @@ -92,8 +92,12 @@ // sanity check, norms should all be 1 assertTrue("Whoops we have norms?", !r.hasNorms("nonorm")); - for (int i = 0; i< norms.length; i++) { - assertEquals(""+i, DEFAULT_NORM, norms[i]); + if (!r.getDisableFakeNorms()) { + for (int i = 0; i< norms.length; i++) { + assertEquals(""+i, DEFAULT_NORM, norms[i]); + } + } else { + assertNull(norms); } r.close(); @@ -110,8 +114,12 @@ norms = r.norms("nonorm"); assertTrue("Whoops we have norms?", !r.hasNorms("nonorm")); - for (int i = 0; i< norms.length; i++) { - assertEquals(""+i, DEFAULT_NORM, norms[i]); + if (!r.getDisableFakeNorms()) { + for (int i = 0; i< norms.length; i++) { + assertEquals(""+i, DEFAULT_NORM, norms[i]); + } + } else { + assertNull(norms); } r.close(); Index: contrib/miscellaneous/src/test/org/apache/lucene/misc/TestLengthNormModifier.java =================================================================== --- contrib/miscellaneous/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (revision 768911) +++ contrib/miscellaneous/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (working copy) @@ -98,9 +98,13 @@ // sanity check, norms should all be 1 assertTrue("Whoops we have norms?", !r.hasNorms("nonorm")); - for (int i = 0; i< norms.length; i++) { + if (!r.getDisableFakeNorms()) { + for (int i = 0; i< norms.length; i++) { assertEquals(""+i, DEFAULT_NORM, norms[i]); - } + } + } else { + assertNull(norms); + } r.close(); @@ -116,9 +120,13 @@ norms = r.norms("nonorm"); assertTrue("Whoops we have norms?", !r.hasNorms("nonorm")); - for (int i = 0; i< norms.length; i++) { + if (!r.getDisableFakeNorms()) { + for (int i = 0; i< norms.length; i++) { assertEquals(""+i, DEFAULT_NORM, norms[i]); - } + } + } else { + assertNull(norms); + } r.close();