Index: solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java =================================================================== --- solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java (revision 1230437) +++ solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java (working copy) @@ -18,6 +18,7 @@ package org.apache.solr.search.function; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.DocValueNorm; import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.similarities.DefaultSimilarity; @@ -346,8 +347,11 @@ FieldInvertState state = new FieldInvertState(); state.setBoost(1.0f); state.setLength(4); + DocValueNorm norm = new DocValueNorm(); + similarity.computeNorm(state, norm); + float nrm = similarity.decodeNormValue(norm.getBytes().bytes[0]); assertQ(req("fl","*,score","q", "{!func}norm(a_t)", "fq","id:2"), - "//float[@name='score']='" + similarity.decodeNormValue(similarity.computeNorm(state)) + "'"); // sqrt(4)==2 and is exactly representable when quantized to a byte + "//float[@name='score']='" + nrm + "'"); // sqrt(4)==2 and is exactly representable when quantized to a byte // test that ord and rord are working on a global index basis, not just // at the segment level (since Lucene 2.9 has switched to per-segment searching) Index: lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (working copy) @@ -330,8 +330,8 @@ //Make everything else 1 so we see the effect of the payload //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue(state.getBoost()); + public void computeNorm(FieldInvertState state, Norm norm) { + norm.setByte(encodeNormValue(state.getBoost())); } @Override Index: lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.SimilarityProvider; +import org.apache.lucene.search.similarities.Similarity.Norm; import org.apache.lucene.search.spans.MultiSpansWrapper; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.search.spans.Spans; @@ -324,9 +325,9 @@ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //Make everything else 1 so we see the effect of the payload //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue(state.getBoost()); + @Override + public void computeNorm(FieldInvertState state, Norm norm) { + norm.setByte(encodeNormValue(state.getBoost())); } @Override Index: lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java (working copy) @@ -151,8 +151,8 @@ } @Override - public byte computeNorm(FieldInvertState state) { - return sim.computeNorm(state); + public void computeNorm(FieldInvertState state, Norm norm) { + sim.computeNorm(state, norm); } @Override Index: lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (working copy) @@ -66,9 +66,9 @@ } @Override - public byte computeNorm(FieldInvertState state) { + public void computeNorm(FieldInvertState state, Norm norm) { // Disable length norm - return encodeNormValue(state.getBoost()); + norm.setByte(encodeNormValue(state.getBoost())); } @Override Index: lucene/src/test/org/apache/lucene/search/JustCompileSearch.java =================================================================== --- lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (working copy) @@ -261,7 +261,7 @@ } @Override - public byte computeNorm(FieldInvertState state) { + public void computeNorm(FieldInvertState state, Norm norm) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } } Index: lucene/src/test/org/apache/lucene/search/TestSimilarity.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestSimilarity.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/search/TestSimilarity.java (working copy) @@ -45,7 +45,7 @@ public float coord(int overlap, int maxOverlap) { return 1.0f; } public Similarity get(String field) { return new DefaultSimilarity() { - @Override public byte computeNorm(FieldInvertState state) { return encodeNormValue(state.getBoost()); } + @Override public void computeNorm(FieldInvertState state, Norm norm) { norm.setByte(encodeNormValue(state.getBoost())); } @Override public float tf(float freq) { return freq; } @Override public float sloppyFreq(int distance) { return 2.0f; } @Override public float idf(int docFreq, int numDocs) { return 1.0f; } Index: lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java (working copy) @@ -111,9 +111,10 @@ } private class Sim1 extends TFIDFSimilarity { + @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue(1f); + public void computeNorm(FieldInvertState state, Norm norm) { + norm.setByte(encodeNormValue(1f)); } @Override @@ -138,9 +139,10 @@ } private class Sim2 extends TFIDFSimilarity { + @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue(10f); + public void computeNorm(FieldInvertState state, Norm norm) { + norm.setByte(encodeNormValue(10f)); } @Override Index: lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (working copy) @@ -117,8 +117,8 @@ } @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue((float) state.getMaxTermFrequency()); + public void computeNorm(FieldInvertState state, Norm norm) { + norm.setByte(encodeNormValue((float) state.getMaxTermFrequency())); } } } Index: lucene/src/test/org/apache/lucene/index/TestNorms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestNorms.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/index/TestNorms.java (working copy) @@ -17,20 +17,13 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Random; - -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -52,8 +45,8 @@ } @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue((float) state.getLength()); + public void computeNorm(FieldInvertState state, Norm norm) { + norm.setByte(encodeNormValue((float) state.getLength())); } } Index: lucene/src/test/org/apache/lucene/index/TestFieldInfos.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (working copy) @@ -141,7 +141,7 @@ try { readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), - random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null); + random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, null); fail("instance should be read only"); } catch (IllegalStateException e) { // expected Index: lucene/src/test/org/apache/lucene/index/TestCustomNorms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCustomNorms.java (revision 0) +++ lucene/src/test/org/apache/lucene/index/TestCustomNorms.java (working copy) @@ -0,0 +1,129 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DocValues.Source; +import org.apache.lucene.index.DocValues.Type; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase; +import org.junit.Before; + +/** + * + */ +public class TestCustomNorms extends LuceneTestCase { + final String testField = "normsTest"; + + @Before + public void setUp() throws Exception { + super.setUp(); + assumeFalse("cannot work with preflex codec", Codec.getDefault().getName().equals("Lucene3x")); + assumeFalse("cannot work with simple text codec", Codec.getDefault().getName().equals("SimpleText")); + + } + + public void testFloatNorms() throws IOException { + + MockDirectoryWrapper dir = newDirectory(); + dir.setCheckIndexOnClose(false); // can't set sim to checkindex yet + IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)); + SimilarityProvider provider = new MySimProvider(); + config.setSimilarityProvider(provider); + RandomIndexWriter writer = new RandomIndexWriter(random, dir, config); + final LineFileDocs docs = new LineFileDocs(random); + int num = 10; //atLeast(100); + for (int i = 0; i < num; i++) { + Document doc = docs.nextDoc(); + float nextFloat = random.nextFloat(); + Field f = new Field(testField, "" + nextFloat, + TextField.TYPE_STORED); + f.setBoost(nextFloat); + + doc.add(f); + writer.addDocument(doc); + doc.removeField(testField); + if (rarely()) { + writer.commit(); + } + } + writer.commit(); + writer.close(); + IndexReader open = new SlowMultiReaderWrapper(IndexReader.open(dir)); + DocValues normValues = open.normValues(testField); + assertNotNull(normValues); + Source source = normValues.getSource(); + assertTrue(source.hasArray()); + assertEquals(Type.FLOAT_32, normValues.type()); + float[] norms = (float[]) source.getArray(); + for (int i = 0; i < open.maxDoc(); i++) { + Document document = open.document(i); + float expected = Float.parseFloat(document.get(testField)); + assertEquals(expected, norms[i], 0.0f); + } + open.close(); + dir.close(); + + } + + public class MySimProvider implements SimilarityProvider { + SimilarityProvider delegate = new DefaultSimilarityProvider(); + + @Override + public float queryNorm(float sumOfSquaredWeights) { + + return delegate.queryNorm(sumOfSquaredWeights); + } + + @Override + public Similarity get(String field) { + if (testField.equals(field)) { + return new FloatEncodingBoostSimilarity(); + } else { + return delegate.get(field); + } + } + + @Override + public float coord(int overlap, int maxOverlap) { + return delegate.coord(overlap, maxOverlap); + } + } + + public static class FloatEncodingBoostSimilarity extends DefaultSimilarity { + + @Override + public void computeNorm(FieldInvertState state, Norm norm) { + float boost = state.getBoost(); + norm.setFloat(boost); + } + } + +} Index: lucene/src/test/org/apache/lucene/index/TestOmitTf.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestOmitTf.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/index/TestOmitTf.java (working copy) @@ -44,8 +44,7 @@ public float coord(int overlap, int maxOverlap) { return 1.0f; } public Similarity get(String field) { return new TFIDFSimilarity() { - - @Override public byte computeNorm(FieldInvertState state) { return encodeNormValue(state.getBoost()); } + @Override public void computeNorm(FieldInvertState state, Norm norm) { norm.setByte(encodeNormValue(state.getBoost())); } @Override public float tf(float freq) { return freq; } @Override public float sloppyFreq(int distance) { return 2.0f; } @Override public float idf(int docFreq, int numDocs) { return 1.0f; } Index: lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java (revision 1230437) +++ lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java (working copy) @@ -102,8 +102,8 @@ class TestSimilarity extends DefaultSimilarity { @Override - public byte computeNorm(FieldInvertState state) { - return (byte) state.getUniqueTermCount(); + public void computeNorm(FieldInvertState state, Norm norm) { + norm.setByte((byte) state.getUniqueTermCount()); } } } Index: lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java (working copy) @@ -77,7 +77,7 @@ @Override protected Type getDocValuesType(FieldInfo info) { - return Type.BYTES_FIXED_STRAIGHT; + return info.getNormType(); } @Override @@ -107,7 +107,7 @@ @Override protected Type getDocValuesType(FieldInfo info) { - return Type.BYTES_FIXED_STRAIGHT; + return info.getNormType(); } public static void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { Index: lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java (working copy) @@ -65,7 +65,7 @@ for (int i = 0; i < size; i++) { String name = input.readString(); - final int fieldNumber = format <= Lucene40FieldInfosWriter.FORMAT_FLEX? input.readInt():i; + final int fieldNumber = input.readInt(); byte bits = input.readByte(); boolean isIndexed = (bits & Lucene40FieldInfosWriter.IS_INDEXED) != 0; boolean storeTermVector = (bits & Lucene40FieldInfosWriter.STORE_TERMVECTOR) != 0; @@ -75,11 +75,7 @@ if ((bits & Lucene40FieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) { indexOptions = IndexOptions.DOCS_ONLY; } else if ((bits & Lucene40FieldInfosWriter.OMIT_POSITIONS) != 0) { - if (format <= Lucene40FieldInfosWriter.FORMAT_OMIT_POSITIONS) { - indexOptions = IndexOptions.DOCS_AND_FREQS; - } else { - throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")"); - } + indexOptions = IndexOptions.DOCS_AND_FREQS; } else { indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; } @@ -93,59 +89,12 @@ hasVectors |= storeTermVector; hasProx |= isIndexed && indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; hasFreq |= isIndexed && indexOptions != IndexOptions.DOCS_ONLY; - DocValues.Type docValuesType = null; - if (format <= Lucene40FieldInfosWriter.FORMAT_FLEX) { - final byte b = input.readByte(); - switch(b) { - case 0: - docValuesType = null; - break; - case 1: - docValuesType = DocValues.Type.VAR_INTS; - break; - case 2: - docValuesType = DocValues.Type.FLOAT_32; - break; - case 3: - docValuesType = DocValues.Type.FLOAT_64; - break; - case 4: - docValuesType = DocValues.Type.BYTES_FIXED_STRAIGHT; - break; - case 5: - docValuesType = DocValues.Type.BYTES_FIXED_DEREF; - break; - case 6: - docValuesType = DocValues.Type.BYTES_VAR_STRAIGHT; - break; - case 7: - docValuesType = DocValues.Type.BYTES_VAR_DEREF; - break; - case 8: - docValuesType = DocValues.Type.FIXED_INTS_16; - break; - case 9: - docValuesType = DocValues.Type.FIXED_INTS_32; - break; - case 10: - docValuesType = DocValues.Type.FIXED_INTS_64; - break; - case 11: - docValuesType = DocValues.Type.FIXED_INTS_8; - break; - case 12: - docValuesType = DocValues.Type.BYTES_FIXED_SORTED; - break; - case 13: - docValuesType = DocValues.Type.BYTES_VAR_SORTED; - break; - - default: - throw new IllegalStateException("unhandled indexValues type " + b); - } - } + // DV Types are packed in one byte + byte val = input.readByte(); + final DocValues.Type docValuesType = getDocValuesType((byte) (val & 0x0F)); + final DocValues.Type normsType = getDocValuesType((byte) ((val >>> 4) & 0x0F)); infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, - omitNorms, storePayloads, indexOptions, docValuesType); + omitNorms, storePayloads, indexOptions, docValuesType, normsType); } if (input.getFilePointer() != input.length()) { @@ -157,6 +106,42 @@ input.close(); } } + + public DocValues.Type getDocValuesType( + final byte b) { + switch(b) { + case 0: + return null; + case 1: + return DocValues.Type.VAR_INTS; + case 2: + return DocValues.Type.FLOAT_32; + case 3: + return DocValues.Type.FLOAT_64; + case 4: + return DocValues.Type.BYTES_FIXED_STRAIGHT; + case 5: + return DocValues.Type.BYTES_FIXED_DEREF; + case 6: + return DocValues.Type.BYTES_VAR_STRAIGHT; + case 7: + return DocValues.Type.BYTES_VAR_DEREF; + case 8: + return DocValues.Type.FIXED_INTS_16; + case 9: + return DocValues.Type.FIXED_INTS_32; + case 10: + return DocValues.Type.FIXED_INTS_64; + case 11: + return DocValues.Type.FIXED_INTS_8; + case 12: + return DocValues.Type.BYTES_FIXED_SORTED; + case 13: + return DocValues.Type.BYTES_VAR_SORTED; + default: + throw new IllegalStateException("unhandled indexValues type " + b); + } + } public static void files(Directory dir, SegmentInfo info, Set files) throws IOException { files.add(IndexFileNames.segmentFileName(info.name, "", Lucene40FieldInfosWriter.FIELD_INFOS_EXTENSION)); Index: lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import org.apache.lucene.codecs.FieldInfosWriter; +import org.apache.lucene.index.DocValues.Type; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; @@ -35,15 +36,11 @@ /** Extension of field infos */ static final String FIELD_INFOS_EXTENSION = "fnm"; - // First used in 2.9; prior to 2.9 there was no format header - static final int FORMAT_START = -2; - // First used in 3.4: omit only positional information - static final int FORMAT_OMIT_POSITIONS = -3; // per-field codec support, records index values for fields - static final int FORMAT_FLEX = -4; + static final int FORMAT_START = -4; // whenever you add a new format, make it 1 smaller (negative version logic)! - static final int FORMAT_CURRENT = FORMAT_FLEX; + static final int FORMAT_CURRENT = FORMAT_START; static final byte IS_INDEXED = 0x1; static final byte STORE_TERMVECTOR = 0x2; @@ -75,60 +72,53 @@ output.writeInt(fi.number); output.writeByte(bits); - final byte b; - - if (!fi.hasDocValues()) { - b = 0; - } else { - switch(fi.getDocValuesType()) { - case VAR_INTS: - b = 1; - break; - case FLOAT_32: - b = 2; - break; - case FLOAT_64: - b = 3; - break; - case BYTES_FIXED_STRAIGHT: - b = 4; - break; - case BYTES_FIXED_DEREF: - b = 5; - break; - case BYTES_VAR_STRAIGHT: - b = 6; - break; - case BYTES_VAR_DEREF: - b = 7; - break; - case FIXED_INTS_16: - b = 8; - break; - case FIXED_INTS_32: - b = 9; - break; - case FIXED_INTS_64: - b = 10; - break; - case FIXED_INTS_8: - b = 11; - break; - case BYTES_FIXED_SORTED: - b = 12; - break; - case BYTES_VAR_SORTED: - b = 13; - break; - default: - throw new IllegalStateException("unhandled indexValues type " + fi.getDocValuesType()); - } - } - output.writeByte(b); + // pack the DV types in one byte + final byte dv = docValuesByte(fi.getDocValuesType()); + final byte nrm = docValuesByte(fi.getNormType()); + assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0; + byte val = (byte) (0xff & ((nrm << 4) | dv)); + output.writeByte(val); } } finally { output.close(); } } + + public byte docValuesByte(Type type) { + if (type == null) { + return 0; + } else { + switch(type) { + case VAR_INTS: + return 1; + case FLOAT_32: + return 2; + case FLOAT_64: + return 3; + case BYTES_FIXED_STRAIGHT: + return 4; + case BYTES_FIXED_DEREF: + return 5; + case BYTES_VAR_STRAIGHT: + return 6; + case BYTES_VAR_DEREF: + return 7; + case FIXED_INTS_16: + return 8; + case FIXED_INTS_32: + return 9; + case FIXED_INTS_64: + return 10; + case FIXED_INTS_8: + return 11; + case BYTES_FIXED_SORTED: + return 12; + case BYTES_VAR_SORTED: + return 13; + default: + throw new IllegalStateException("unhandled indexValues type " + type); + } + } + } } Index: lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java (revision 0) +++ lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java (working copy) @@ -0,0 +1,49 @@ +package org.apache.lucene.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.FieldInfosReader; +import org.apache.lucene.codecs.FieldInfosWriter; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; + +/** + * @lucene.experimental + */ +public class Lucene3xFieldInfosFormat extends FieldInfosFormat { + private final FieldInfosReader reader = new Lucene3xFieldInfosReader(); + + @Override + public FieldInfosReader getFieldInfosReader() throws IOException { + return reader; + } + + @Override + public FieldInfosWriter getFieldInfosWriter() throws IOException { + throw new IllegalArgumentException("this codec can only be used for reading"); + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + Lucene3xFieldInfosReader.files(dir, info, files); + } +} Property changes on: lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosFormat.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java (revision 0) +++ lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java (working copy) @@ -0,0 +1,123 @@ +package org.apache.lucene.codecs.lucene3x; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.codecs.FieldInfosReader; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocValues.Type; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; + +/** + * @lucene.experimental + */ +public class Lucene3xFieldInfosReader extends FieldInfosReader { + /** Extension of field infos */ + static final String FIELD_INFOS_EXTENSION = "fnm"; + + // First used in 2.9; prior to 2.9 there was no format header + static final int FORMAT_START = -2; + // First used in 3.4: omit only positional information + static final int FORMAT_OMIT_POSITIONS = -3; + static final int FORMAT_MINIMUM = FORMAT_START; + static final int FORMAT_CURRENT = FORMAT_OMIT_POSITIONS; + static final byte IS_INDEXED = 0x1; + static final byte STORE_TERMVECTOR = 0x2; + static final byte OMIT_NORMS = 0x10; + static final byte STORE_PAYLOADS = 0x20; + static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40; + static final byte OMIT_POSITIONS = -128; + + @Override + public FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException { + final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION); + IndexInput input = directory.openInput(fileName, iocontext); + + boolean hasVectors = false; + boolean hasFreq = false; + boolean hasProx = false; + + try { + final int format = input.readVInt(); + + if (format > FORMAT_MINIMUM) { + throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT); + } + if (format < FORMAT_CURRENT) { + throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT); + } + + final int size = input.readVInt(); //read in the size + FieldInfo infos[] = new FieldInfo[size]; + + for (int i = 0; i < size; i++) { + String name = input.readString(); + final int fieldNumber = i; + byte bits = input.readByte(); + boolean isIndexed = (bits & IS_INDEXED) != 0; + boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; + boolean omitNorms = (bits & OMIT_NORMS) != 0; + boolean storePayloads = (bits & STORE_PAYLOADS) != 0; + final IndexOptions indexOptions; + if ((bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0) { + indexOptions = IndexOptions.DOCS_ONLY; + } else if ((bits & OMIT_POSITIONS) != 0) { + if (format <= FORMAT_OMIT_POSITIONS) { + indexOptions = IndexOptions.DOCS_AND_FREQS; + } else { + throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")"); + } + } else { + indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + } + + // LUCENE-3027: past indices were able to write + // storePayloads=true when omitTFAP is also true, + // which is invalid. We correct that, here: + if (indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { + storePayloads = false; + } + hasVectors |= storeTermVector; + hasProx |= isIndexed && indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + hasFreq |= isIndexed && indexOptions != IndexOptions.DOCS_ONLY; + infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, + omitNorms, storePayloads, indexOptions, null, isIndexed && !omitNorms? Type.BYTES_VAR_STRAIGHT : null); + } + + if (input.getFilePointer() != input.length()) { + throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")"); + } + return new FieldInfos(infos, hasFreq, hasProx, hasVectors); + } finally { + input.close(); + } + } + + public static void files(Directory dir, SegmentInfo info, Set files) throws IOException { + files.add(IndexFileNames.segmentFileName(info.name, "", FIELD_INFOS_EXTENSION)); + } +} Property changes on: lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xCodec.java (working copy) @@ -30,7 +30,6 @@ import org.apache.lucene.codecs.SegmentInfosFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.TermVectorsFormat; -import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat; import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfosFormat; import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat; import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat; @@ -56,7 +55,7 @@ private final TermVectorsFormat vectorsFormat = new Lucene40TermVectorsFormat(); // TODO: this should really be a different impl - private final FieldInfosFormat fieldInfosFormat = new Lucene40FieldInfosFormat(); + private final FieldInfosFormat fieldInfosFormat = new Lucene3xFieldInfosFormat(); // TODO: this should really be a different impl // also if we want preflex to *really* be read-only it should throw exception for the writer? Index: lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java =================================================================== --- lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java (working copy) @@ -86,18 +86,19 @@ SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch, NORMS); boolean omitNorms = !Boolean.parseBoolean(readString(NORMS.length, scratch)); - + SimpleTextUtil.readLine(input, scratch); + assert StringHelper.startsWith(scratch, NORMS_TYPE); + String nrmType = readString(NORMS_TYPE.length, scratch); + final DocValues.Type normsType = docValuesType(nrmType); + + SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch, DOCVALUES); String dvType = readString(DOCVALUES.length, scratch); - final DocValues.Type docValuesType; + final DocValues.Type docValuesType = docValuesType(dvType); - if ("false".equals(dvType)) { - docValuesType = null; - } else { - docValuesType = DocValues.Type.valueOf(dvType); - } + SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch, INDEXOPTIONS); IndexOptions indexOptions = IndexOptions.valueOf(readString(INDEXOPTIONS.length, scratch)); @@ -107,7 +108,7 @@ hasFreq |= isIndexed && indexOptions != IndexOptions.DOCS_ONLY; infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, - omitNorms, storePayloads, indexOptions, docValuesType); + omitNorms, storePayloads, indexOptions, docValuesType, normsType); } if (input.getFilePointer() != input.length()) { @@ -119,6 +120,14 @@ input.close(); } } + + public DocValues.Type docValuesType(String dvType) { + if ("false".equals(dvType)) { + return null; + } else { + return DocValues.Type.valueOf(dvType); + } + } private String readString(int offset, BytesRef scratch) { return new String(scratch.bytes, scratch.offset+offset, scratch.length-offset, IOUtils.CHARSET_UTF_8); Index: lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import org.apache.lucene.codecs.FieldInfosWriter; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; @@ -48,6 +49,7 @@ static final BytesRef STORETVOFF = new BytesRef(" term vector offsets "); static final BytesRef PAYLOADS = new BytesRef(" payloads "); static final BytesRef NORMS = new BytesRef(" norms "); + static final BytesRef NORMS_TYPE = new BytesRef(" norms type "); static final BytesRef DOCVALUES = new BytesRef(" doc values "); static final BytesRef INDEXOPTIONS = new BytesRef(" index options "); @@ -88,12 +90,12 @@ SimpleTextUtil.write(out, Boolean.toString(!fi.omitNorms), scratch); SimpleTextUtil.writeNewline(out); + SimpleTextUtil.write(out, NORMS_TYPE); + SimpleTextUtil.write(out, getDocValuesType(fi.getNormType()), scratch); + SimpleTextUtil.writeNewline(out); + SimpleTextUtil.write(out, DOCVALUES); - if (!fi.hasDocValues()) { - SimpleTextUtil.write(out, "false", scratch); - } else { - SimpleTextUtil.write(out, fi.getDocValuesType().toString(), scratch); - } + SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch); SimpleTextUtil.writeNewline(out); SimpleTextUtil.write(out, INDEXOPTIONS); @@ -104,4 +106,8 @@ out.close(); } } + + private static String getDocValuesType(DocValues.Type type) { + return type == null ? "false" : type.toString(); + } } Index: lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java =================================================================== --- lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (working copy) @@ -122,10 +122,11 @@ } } + @Override - public final byte computeNorm(FieldInvertState state) { + public final void computeNorm(FieldInvertState state, Norm norm) { final int numTerms = discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength(); - return encodeNormValue(state.getBoost(), numTerms); + norm.setByte(encodeNormValue(state.getBoost(), numTerms)); } public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) { Index: lucene/src/java/org/apache/lucene/search/similarities/Similarity.java =================================================================== --- lucene/src/java/org/apache/lucene/search/similarities/Similarity.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/search/similarities/Similarity.java (working copy) @@ -21,6 +21,8 @@ import java.io.IOException; import org.apache.lucene.document.DocValuesField; // javadoc +import org.apache.lucene.index.DocValue; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader; // javadoc import org.apache.lucene.index.IndexReader.AtomicReaderContext; @@ -36,7 +38,6 @@ import org.apache.lucene.search.spans.SpanQuery; // javadoc import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.SmallFloat; // javadoc -import org.apache.lucene.util.TermContext; /** @@ -55,8 +56,8 @@ * query-time. *

* - * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing - * the Similarity implementation to return a per-document byte for the field that will + * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState, Norm)}, allowing + * the Similarity implementation to set a per-document value for the field that will * be later accessible via {@link IndexReader#normValues(String)}. Lucene makes no assumption * about what is in this byte, but it is most useful for encoding length normalization * information. @@ -109,23 +110,24 @@ * @lucene.experimental */ public abstract class Similarity { + /** * Computes the normalization value for a field, given the accumulated * state of term processing for this field (see {@link FieldInvertState}). * - *

Implementations should calculate a byte value based on the field - * state and then return that value. + *

Implementations should calculate a norm value based on the field + * state and set that value to the given {@link Norm}. * *

Matches in longer fields are less precise, so implementations of this - * method usually return smaller values when state.getLength() is large, + * method usually set smaller values when state.getLength() is large, * and larger values when state.getLength() is small. * * @lucene.experimental * * @param state current processing state for this field - * @return the calculated byte norm + * @param norm holds the computed norm value when this method returns */ - public abstract byte computeNorm(FieldInvertState state); + public abstract void computeNorm(FieldInvertState state, Norm norm); /** * Compute any collection-level stats (e.g. IDF, average document length, etc) needed for scoring a query. @@ -229,4 +231,16 @@ */ public abstract void normalize(float queryNorm, float topLevelBoost); } + + // nocommit javadoc + public static abstract class Norm { + public abstract void setFloat(float norm); + public abstract void setFloat(double norm); + public abstract void setInt(short norm); + public abstract void setInt(int norm); + public abstract void setInt(long norm); + public abstract void setInt(byte norm); + public abstract void setByte(byte norm); + public abstract void setBytes(BytesRef norm); + } } Index: lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java =================================================================== --- lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java (working copy) @@ -40,8 +40,8 @@ } @Override - public byte computeNorm(FieldInvertState state) { - return sims[0].computeNorm(state); + public void computeNorm(FieldInvertState state, Norm norm) { + sims[0].computeNorm(state, norm); } @Override Index: lucene/src/java/org/apache/lucene/search/similarities/package.html =================================================================== --- lucene/src/java/org/apache/lucene/search/similarities/package.html (revision 1230437) +++ lucene/src/java/org/apache/lucene/search/similarities/package.html (working copy) @@ -155,7 +155,7 @@ matching term occurs. In these cases people have overridden Similarity to return 1 from the tf() method.

  • Changing Length Normalization — By overriding - {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState state)}, + {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState state, Norm)}, it is possible to discount how the length of a field contributes to a score. In {@link org.apache.lucene.search.similarities.DefaultSimilarity}, lengthNorm = 1 / (numTerms in field)^0.5, but if one changes this to be Index: lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java =================================================================== --- lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (working copy) @@ -234,13 +234,13 @@ /** Encodes the document length in the same way as {@link TFIDFSimilarity}. */ @Override - public byte computeNorm(FieldInvertState state) { + public void computeNorm(FieldInvertState state, Norm norm) { final float numTerms; if (discountOverlaps) numTerms = state.getLength() - state.getNumOverlap(); else numTerms = state.getLength() / state.getBoost(); - return encodeNormValue(state.getBoost(), numTerms); + norm.setByte(encodeNormValue(state.getBoost(), numTerms)); } /** Decodes a normalization factor (document length) stored in an index. Index: lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java =================================================================== --- lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java (working copy) @@ -22,14 +22,12 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.SmallFloat; Index: lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java =================================================================== --- lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java (working copy) @@ -22,7 +22,7 @@ /** Expert: Default scoring implementation. */ public class DefaultSimilarity extends TFIDFSimilarity { - + /** Implemented as * state.getBoost()*lengthNorm(numTerms), where * numTerms is {@link FieldInvertState#getLength()} if {@link @@ -32,13 +32,13 @@ * * @lucene.experimental */ @Override - public byte computeNorm(FieldInvertState state) { + public void computeNorm(FieldInvertState state, Norm norm) { final int numTerms; if (discountOverlaps) numTerms = state.getLength() - state.getNumOverlap(); else numTerms = state.getLength(); - return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)))); + norm.setByte(encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))))); } /** Implemented as sqrt(freq). */ Index: lucene/src/java/org/apache/lucene/index/FieldInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfos.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/index/FieldInfos.java (working copy) @@ -268,7 +268,7 @@ */ synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector, boolean omitNorms) { - addOrUpdate(name, isIndexed, storeTermVector, omitNorms, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null); + addOrUpdate(name, isIndexed, storeTermVector, omitNorms, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, null); } /** If the field is not yet known, adds it. If it is known, checks to make @@ -284,8 +284,8 @@ * @param indexOptions if term freqs should be omitted for this field */ synchronized public FieldInfo addOrUpdate(String name, boolean isIndexed, boolean storeTermVector, - boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) { - return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues); + boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) { + return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType); } // NOTE: this method does not carry over termVector @@ -301,19 +301,19 @@ // be updated by maybe FreqProxTermsWriterPerField: return addOrUpdateInternal(name, -1, fieldType.indexed(), false, fieldType.omitNorms(), false, - fieldType.indexOptions(), null); + fieldType.indexOptions(), null, null); } synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed, boolean storeTermVector, - boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) { + boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normType) { if (globalFieldNumbers == null) { throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos"); } FieldInfo fi = fieldInfo(name); if (fi == null) { final int fieldNumber = nextFieldNumber(name, preferredFieldNumber); - fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues); + fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType); } else { fi.update(isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions); fi.setDocValuesType(docValues); @@ -326,7 +326,7 @@ // IMPORTANT - reuse the field number if possible for consistent field numbers across segments return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed, fi.storeTermVector, fi.omitNorms, fi.storePayloads, - fi.indexOptions, fi.getDocValuesType()); + fi.indexOptions, fi.getDocValuesType(), fi.getNormType()); } /* @@ -334,12 +334,12 @@ */ private FieldInfo addInternal(String name, int fieldNumber, boolean isIndexed, boolean storeTermVector, boolean omitNorms, boolean storePayloads, - IndexOptions indexOptions, DocValues.Type docValuesType) { + IndexOptions indexOptions, DocValues.Type docValuesType, DocValues.Type normType) { // don't check modifiable here since we use that to initially build up FIs if (globalFieldNumbers != null) { globalFieldNumbers.setIfNotSet(fieldNumber, name); } - final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType); + final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, normType); putInternal(fi); return fi; } Index: lucene/src/java/org/apache/lucene/index/NormsConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/NormsConsumer.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/index/NormsConsumer.java (working copy) @@ -24,9 +24,8 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.NormsFormat; import org.apache.lucene.codecs.PerDocConsumer; -import org.apache.lucene.document.DocValuesField; import org.apache.lucene.index.DocValues.Type; -import org.apache.lucene.util.BytesRef; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.IOUtils; // TODO FI: norms could actually be stored as doc store @@ -71,14 +70,19 @@ if (!fi.omitNorms) { if (toWrite != null && toWrite.initialized()) { anythingFlushed = true; - toWrite.flush(state.numDocs); + final Type type = toWrite.flush(state.numDocs); + fi.setNormValueType(type); } else if (fi.isIndexed) { anythingFlushed = true; - final DocValuesConsumer valuesConsumer = newConsumer(new PerDocWriteState(state), fi); - final DocValuesField value = new DocValuesField(""); - value.setBytes(new BytesRef(new byte[] {0x00}), Type.BYTES_FIXED_STRAIGHT); - valuesConsumer.add(state.numDocs-1, value); + // the crazy exception case: + Similarity sim = dwpt.parent.similarityProvider.get(fi.name); + DocValueNorm norm = new DocValueNorm(); + sim.computeNorm(bogusState, norm); + assert norm.type() != null; + final DocValuesConsumer valuesConsumer = newConsumer(new PerDocWriteState(state), fi, norm.type()); + valuesConsumer.add(state.numDocs-1, norm); valuesConsumer.finish(state.numDocs); + fi.setNormValueType(norm.type()); } } } @@ -110,12 +114,13 @@ } DocValuesConsumer newConsumer(PerDocWriteState perDocWriteState, - FieldInfo fieldInfo) throws IOException { + FieldInfo fieldInfo, Type type) throws IOException { if (consumer == null) { consumer = normsFormat.docsConsumer(perDocWriteState); } - DocValuesConsumer addValuesField = consumer.addValuesField( - Type.BYTES_FIXED_STRAIGHT, fieldInfo); + DocValuesConsumer addValuesField = consumer.addValuesField(type, fieldInfo); return addValuesField; } + + static final FieldInvertState bogusState = new FieldInvertState(1,1,1,1,1); } Index: lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java =================================================================== --- lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java (working copy) @@ -18,10 +18,8 @@ import java.io.IOException; import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.document.DocValuesField; import org.apache.lucene.index.DocValues.Type; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.util.BytesRef; public class NormsConsumerPerField extends InvertedDocEndConsumerPerField implements Comparable { private final FieldInfo fieldInfo; @@ -29,9 +27,9 @@ private final Similarity similarity; private final FieldInvertState fieldState; private DocValuesConsumer consumer; - private final DocValuesField value = new DocValuesField(""); - private final BytesRef spare = new BytesRef(1); + private final DocValueNorm norm; private final NormsConsumer parent; + private Type initType; public NormsConsumerPerField(final DocInverterPerField docInverterPerField, final FieldInfo fieldInfo, NormsConsumer parent) { this.fieldInfo = fieldInfo; @@ -39,10 +37,9 @@ docState = docInverterPerField.docState; fieldState = docInverterPerField.fieldState; similarity = docState.similarityProvider.get(fieldInfo.name); - spare.length = 1; - spare.offset = 0; + norm = new DocValueNorm(); + } - } @Override public int compareTo(NormsConsumerPerField other) { return fieldInfo.name.compareTo(other.fieldInfo.name); @@ -51,23 +48,26 @@ @Override void finish() throws IOException { if (fieldInfo.isIndexed && !fieldInfo.omitNorms) { - DocValuesConsumer consumer = getConsumer(); - spare.bytes[0] = similarity.computeNorm(fieldState); - value.setBytes(spare, Type.BYTES_FIXED_STRAIGHT); - consumer.add(docState.docID, value); - + similarity.computeNorm(fieldState, norm); + assert norm.type() != null; + DocValuesConsumer consumer = getConsumer(norm.type()); + consumer.add(docState.docID, norm); + norm.reset(); } } - void flush(int docCount) throws IOException { + Type flush(int docCount) throws IOException { assert initialized(); consumer.finish(docCount); + return initType; } - private DocValuesConsumer getConsumer() throws IOException { + private DocValuesConsumer getConsumer(Type type) throws IOException { if (consumer == null) { - consumer = parent.newConsumer(docState.docWriter.newPerDocWriteState(""), fieldInfo); + consumer = parent.newConsumer(docState.docWriter.newPerDocWriteState(""), fieldInfo, type); + this.initType = type; } + assert initType == type; return consumer; } Index: lucene/src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentMerger.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/index/SegmentMerger.java (working copy) @@ -191,10 +191,19 @@ } private void mergeFieldInfos() throws IOException { + mergeDocValuesFieldInfos(); + mergeNormFieldInfos(); + // write the merged infos + FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat() + .getFieldInfosWriter(); + fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context); + } + + public void mergeDocValuesFieldInfos() throws IOException { // mapping from all docvalues fields found to their promoted types // this is because FieldInfos does not store the valueSize Map docValuesTypes = new HashMap(); - + for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) { final IndexReader reader = readerAndLiveDocs.reader; FieldInfos readerFieldInfos = reader.getFieldInfos(); @@ -222,10 +231,40 @@ } } } + } + + public void mergeNormFieldInfos() throws IOException { + // mapping from all docvalues fields found to their promoted types + // this is because FieldInfos does not store the valueSize + Map normValuesTypes = new HashMap(); + + for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) { + final IndexReader reader = readerAndLiveDocs.reader; + FieldInfos readerFieldInfos = reader.getFieldInfos(); + for (FieldInfo fi : readerFieldInfos) { + FieldInfo merged = mergeState.fieldInfos.add(fi); + // update the type promotion mapping for this reader + if (fi.isIndexed && !fi.omitNorms) { + TypePromoter previous = normValuesTypes.get(merged); + normValuesTypes.put(merged, mergeDocValuesType(previous, reader.normValues(fi.name))); + } + } + } - // write the merged infos - FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter(); - fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context); + // update any promoted doc values types: + for (Map.Entry e : normValuesTypes.entrySet()) { + FieldInfo fi = e.getKey(); + TypePromoter promoter = e.getValue(); + if (promoter == null) { + fi.resetNormValueType(null); + } else { + assert promoter != TypePromoter.getIdentityPromoter(); + if (fi.getDocValuesType() != promoter.type()) { + // reset the type if we got promoted + fi.resetNormValueType(promoter.type()); + } + } + } } /** Index: lucene/src/java/org/apache/lucene/index/FieldInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfo.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/index/FieldInfo.java (working copy) @@ -1,6 +1,8 @@ package org.apache.lucene.index; +import org.apache.lucene.index.DocValues.Type; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -29,6 +31,7 @@ // True if any document indexed term vectors public boolean storeTermVector; + private DocValues.Type normType; public boolean omitNorms; // omit norms associated with indexed fields public IndexOptions indexOptions; public boolean storePayloads; // whether this field stores payloads together with term positions @@ -51,7 +54,7 @@ * @lucene.experimental */ public FieldInfo(String name, boolean isIndexed, int number, boolean storeTermVector, - boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) { + boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues, DocValues.Type normsType) { this.name = name; this.isIndexed = isIndexed; this.number = number; @@ -61,11 +64,13 @@ this.storePayloads = storePayloads; this.omitNorms = omitNorms; this.indexOptions = indexOptions; + this.normType = !omitNorms ? normsType : null; } else { // for non-indexed fields, leave defaults this.storeTermVector = false; this.storePayloads = false; this.omitNorms = false; this.indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + this.normType = null; } assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !storePayloads; } @@ -73,7 +78,7 @@ @Override public Object clone() { return new FieldInfo(name, isIndexed, number, storeTermVector, - omitNorms, storePayloads, indexOptions, docValues); + omitNorms, storePayloads, indexOptions, docValues, normType); } // should only be called by FieldInfos#addOrUpdate @@ -120,8 +125,24 @@ public DocValues.Type getDocValuesType() { return docValues; } + + public DocValues.Type getNormType() { + return normType; + } public void setStoreTermVectors() { storeTermVector = true; } + + public void setNormValueType(Type type) { + if (normType == null) { + normType = type; + } + } + + public void resetNormValueType(DocValues.Type v) { + if (normType != null) { + normType = v; + } + } } Index: lucene/src/java/org/apache/lucene/index/DocValueNorm.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocValueNorm.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/DocValueNorm.java (working copy) @@ -0,0 +1,133 @@ +package org.apache.lucene.index; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.util.Comparator; + +import org.apache.lucene.index.DocValues.Type; +import org.apache.lucene.search.similarities.Similarity.Norm; +import org.apache.lucene.util.BytesRef; + +/** + * + * @lucene.experimental + * @lucene.internal + */ +public class DocValueNorm extends Norm implements DocValue { + private double floatValue; + private long intValue; + private BytesRef bytesValue; + private Type type; + + @Override + public void setFloat(float norm) { + setType(Type.FLOAT_32); + floatValue = norm; + } + + private void setType(Type type) { + this.type = type; + } + + @Override + public void setFloat(double norm) { + setType(Type.FLOAT_64); + floatValue = norm; + } + + @Override + public void setInt(short norm) { + setType(Type.FIXED_INTS_16); + intValue = norm; + } + + @Override + public void setInt(int norm) { + setType(Type.FIXED_INTS_32); + intValue = norm; + } + + @Override + public void setInt(long norm) { + setType(Type.FIXED_INTS_64); + intValue = norm; + } + + @Override + public void setByte(byte norm) { + setType(Type.BYTES_FIXED_STRAIGHT); + if (bytesValue == null) { + bytesValue = new BytesRef(new byte[1]); + } + if (bytesValue.bytes.length < 1) { + bytesValue.grow(1); + } + bytesValue.offset = 0; + bytesValue.length = 1; + bytesValue.bytes[0] = norm; + + } + + @Override + public void setInt(byte norm) { + setType(Type.FIXED_INTS_8); + intValue = norm; + } + + @Override + public void setBytes(BytesRef norm) { + setType(Type.BYTES_FIXED_STRAIGHT); + if (bytesValue == null) { + bytesValue = new BytesRef(); + } + this.bytesValue.copyBytes(norm); + } + + @Override + public BytesRef getBytes() { + return bytesValue; + } + + @Override + public Comparator bytesComparator() { + return null; + } + + @Override + public double getFloat() { + return floatValue; + } + + @Override + public long getInt() { + return intValue; + } + + public Type type() { + return type; + } + + public void reset() { + intValue = 0; + floatValue = 0; + if (bytesValue != null) { + bytesValue.offset = 0; + bytesValue.length = 0; + } + type = null; + } + +} \ No newline at end of file Index: lucene/src/java/org/apache/lucene/document/Field.java =================================================================== --- lucene/src/java/org/apache/lucene/document/Field.java (revision 1230437) +++ lucene/src/java/org/apache/lucene/document/Field.java (working copy) @@ -236,13 +236,13 @@ * document. * *

    The boost is used to compute the norm factor for the field. By - * default, in the {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState)} method, + * default, in the {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState, Norm)} method, * the boost value is multiplied by the length normalization factor and then * rounded by {@link org.apache.lucene.search.similarities.DefaultSimilarity#encodeNormValue(float)} before it is stored in the * index. One should attempt to ensure that this product does not overflow * the range of that encoding. * - * @see org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState) + * @see org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState, Norm) * @see org.apache.lucene.search.similarities.DefaultSimilarity#encodeNormValue(float) */ public void setBoost(float boost) { Index: lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosWriter.java =================================================================== --- lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosWriter.java (revision 0) +++ lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosWriter.java (working copy) @@ -0,0 +1,91 @@ +package org.apache.lucene.codecs.preflexrw; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; + +import org.apache.lucene.codecs.FieldInfosWriter; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; + +/** + * @lucene.internal + * @lucene.experimental + */ +public class PreFlexFieldInfosWriter extends FieldInfosWriter { + // TODO move to test-framework preflex RW? + + /** Extension of field infos */ + static final String FIELD_INFOS_EXTENSION = "fnm"; + + // First used in 2.9; prior to 2.9 there was no format header + static final int FORMAT_START = -2; + // First used in 3.4: omit only positional information + static final int FORMAT_OMIT_POSITIONS = -3; + + static final int FORMAT_PREFLEX_RW = Integer.MIN_VALUE; + + // whenever you add a new format, make it 1 smaller (negative version logic)! + static final int FORMAT_CURRENT = FORMAT_OMIT_POSITIONS; + + static final byte IS_INDEXED = 0x1; + static final byte STORE_TERMVECTOR = 0x2; + static final byte OMIT_NORMS = 0x10; + static final byte STORE_PAYLOADS = 0x20; + static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40; + static final byte OMIT_POSITIONS = -128; + + @Override + public void write(Directory directory, String segmentName, FieldInfos infos, IOContext context) throws IOException { + final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION); + IndexOutput output = directory.createOutput(fileName, context); + try { + output.writeVInt(FORMAT_PREFLEX_RW); + output.writeVInt(infos.size()); + for (FieldInfo fi : infos) { + assert fi.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !fi.storePayloads; + byte bits = 0x0; + if (fi.isIndexed) bits |= IS_INDEXED; + if (fi.storeTermVector) bits |= STORE_TERMVECTOR; + if (fi.omitNorms) bits |= OMIT_NORMS; + if (fi.storePayloads) bits |= STORE_PAYLOADS; + if (fi.indexOptions == IndexOptions.DOCS_ONLY) { + bits |= OMIT_TERM_FREQ_AND_POSITIONS; + } else if (fi.indexOptions == IndexOptions.DOCS_AND_FREQS) { + bits |= OMIT_POSITIONS; + } + output.writeString(fi.name); + /* + * we need to write the field number since IW tries + * to stabelize the field numbers across segments so the + * FI ordinal is not necessarily equivalent to the field number + */ + output.writeInt(fi.number); + output.writeByte(bits); + + } + } finally { + output.close(); + } + } + +} Property changes on: lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosWriter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java =================================================================== --- lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java (revision 1230437) +++ lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.NormsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene3x.Lucene3xCodec; @@ -29,6 +30,7 @@ public class PreFlexRWCodec extends Lucene3xCodec { private final PostingsFormat postings = new PreFlexRWPostingsFormat(); private final NormsFormat norms = new PreFlexRWNormsFormat(); + private final FieldInfosFormat fieldInfos = new PreFlexFieldInfosFormat(); @Override public PostingsFormat postingsFormat() { @@ -47,4 +49,13 @@ return super.normsFormat(); } } + + @Override + public FieldInfosFormat fieldInfosFormat() { + if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { + return fieldInfos; + } else { + return super.fieldInfosFormat(); + } + } } Index: lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosFormat.java =================================================================== --- lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosFormat.java (revision 0) +++ lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosFormat.java (working copy) @@ -0,0 +1,41 @@ +package org.apache.lucene.codecs.preflexrw; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; + +import org.apache.lucene.codecs.FieldInfosReader; +import org.apache.lucene.codecs.FieldInfosWriter; +import org.apache.lucene.codecs.lucene3x.Lucene3xFieldInfosFormat; + +/** + * + * @lucene.internal + * @lucene.experimental + */ +public class PreFlexFieldInfosFormat extends Lucene3xFieldInfosFormat { + + @Override + public FieldInfosReader getFieldInfosReader() throws IOException { + return new PreFlexFieldInfosReader(); + } + + @Override + public FieldInfosWriter getFieldInfosWriter() throws IOException { + return new PreFlexFieldInfosWriter(); + } +} Index: lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosReader.java =================================================================== --- lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosReader.java (revision 0) +++ lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosReader.java (working copy) @@ -0,0 +1,110 @@ +package org.apache.lucene.codecs.preflexrw; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.codecs.FieldInfosReader; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocValues.Type; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; + +/** + * @lucene.internal + * @lucene.experimental + */ +public class PreFlexFieldInfosReader extends FieldInfosReader { + static final int FORMAT_MINIMUM = PreFlexFieldInfosWriter.FORMAT_START; + + @Override + public FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException { + final String fileName = IndexFileNames.segmentFileName(segmentName, "", PreFlexFieldInfosWriter.FIELD_INFOS_EXTENSION); + IndexInput input = directory.openInput(fileName, iocontext); + + boolean hasVectors = false; + boolean hasFreq = false; + boolean hasProx = false; + + try { + final int format = input.readVInt(); + + if (format > FORMAT_MINIMUM) { + throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, PreFlexFieldInfosWriter.FORMAT_CURRENT); + } + if (format < PreFlexFieldInfosWriter.FORMAT_CURRENT && format != PreFlexFieldInfosWriter.FORMAT_PREFLEX_RW) { + throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, PreFlexFieldInfosWriter.FORMAT_CURRENT); + } + + final int size = input.readVInt(); //read in the size + FieldInfo infos[] = new FieldInfo[size]; + + for (int i = 0; i < size; i++) { + String name = input.readString(); + final int fieldNumber = format == PreFlexFieldInfosWriter.FORMAT_PREFLEX_RW ? input.readInt() : i; + byte bits = input.readByte(); + boolean isIndexed = (bits & PreFlexFieldInfosWriter.IS_INDEXED) != 0; + boolean storeTermVector = (bits & PreFlexFieldInfosWriter.STORE_TERMVECTOR) != 0; + boolean omitNorms = (bits & PreFlexFieldInfosWriter.OMIT_NORMS) != 0; + boolean storePayloads = (bits & PreFlexFieldInfosWriter.STORE_PAYLOADS) != 0; + final IndexOptions indexOptions; + if ((bits & PreFlexFieldInfosWriter.OMIT_TERM_FREQ_AND_POSITIONS) != 0) { + indexOptions = IndexOptions.DOCS_ONLY; + } else if ((bits & PreFlexFieldInfosWriter.OMIT_POSITIONS) != 0) { + if (format <= PreFlexFieldInfosWriter.FORMAT_OMIT_POSITIONS) { + indexOptions = IndexOptions.DOCS_AND_FREQS; + } else { + throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")"); + } + } else { + indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + } + + // LUCENE-3027: past indices were able to write + // storePayloads=true when omitTFAP is also true, + // which is invalid. We correct that, here: + if (indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { + storePayloads = false; + } + hasVectors |= storeTermVector; + hasProx |= isIndexed && indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + hasFreq |= isIndexed && indexOptions != IndexOptions.DOCS_ONLY; + infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, + omitNorms, storePayloads, indexOptions, null, isIndexed && !omitNorms? Type.BYTES_VAR_STRAIGHT : null); + } + + if (input.getFilePointer() != input.length()) { + throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")"); + } + return new FieldInfos(infos, hasFreq, hasProx, hasVectors); + } finally { + input.close(); + } + } + + public static void files(Directory dir, SegmentInfo info, Set files) throws IOException { + files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexFieldInfosWriter.FIELD_INFOS_EXTENSION)); + } +} Property changes on: lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexFieldInfosReader.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (revision 1230437) +++ lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (working copy) @@ -24,12 +24,24 @@ import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.index.DocValueNorm; import org.apache.lucene.index.FieldInvertState; + /** * Test of the SweetSpotSimilarity */ public class SweetSpotSimilarityTest extends LuceneTestCase { + + public static float computeAndDecodeNorm(SweetSpotSimilarity decode, Similarity encode, FieldInvertState state) { + return decode.decodeNormValue(computeAndGetNorm(encode, state)); + } + + public static byte computeAndGetNorm(Similarity s, FieldInvertState state) { + DocValueNorm norm = new DocValueNorm(); + s.computeNorm(state, norm); + return norm.getBytes().bytes[0]; + } public void testSweetSpotComputeNorm() { @@ -45,9 +57,13 @@ invertState.setBoost(1.0f); for (int i = 1; i < 1000; i++) { invertState.setLength(i); + DocValueNorm lNorm = new DocValueNorm(); + DocValueNorm rNorm = new DocValueNorm(); + d.computeNorm(invertState, lNorm); + s.computeNorm(invertState, rNorm); assertEquals("base case: i="+i, - d.computeNorm(invertState), - s.computeNorm(invertState), + computeAndGetNorm(d, invertState), + computeAndGetNorm(s, invertState), 0.0f); } @@ -59,15 +75,15 @@ invertState.setLength(i); assertEquals("3,10: spot i="+i, 1.0f, - ss.decodeNormValue(s.computeNorm(invertState)), + computeAndDecodeNorm(ss, ss, invertState), 0.0f); } for (int i = 10; i < 1000; i++) { invertState.setLength(i-9); - final byte normD = d.computeNorm(invertState); + final byte normD = computeAndGetNorm(d, invertState); invertState.setLength(i); - final byte normS = s.computeNorm(invertState); + final byte normS = computeAndGetNorm(s, invertState); assertEquals("3,10: 10