Index: src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocumentInformation.java =================================================================== --- src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocumentInformation.java (revision 823249) +++ src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocumentInformation.java (working copy) @@ -65,6 +65,7 @@ private byte[][] payloads; + private int freq; private int[] termPositions; private InstantiatedTerm term; private InstantiatedDocument document; @@ -136,4 +137,12 @@ void setTermOffsets(TermVectorOffsetInfo[] termOffsets) { this.termOffsets = termOffsets; } + + public int getFreq() { + return freq; + } + + public void setFreq(int freq) { + this.freq = freq; + } } Index: src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java =================================================================== --- src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (revision 823249) +++ src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (working copy) @@ -32,6 +32,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermFreqVector; import org.apache.lucene.index.TermPositionVector; import org.apache.lucene.index.TermPositions; @@ -189,16 +190,6 @@ } document.setDocumentNumber(i); documentsByNumber[i] = document; - for (Fieldable field : document.getDocument().getFields()) { - if (fields == null || fields.contains(field.name())) { - if (field.isTermVectorStored()) { - if (document.getVectorSpace() == null) { - document.setVectorSpace(new HashMap>()); - } - document.getVectorSpace().put(field.name(), new ArrayList()); - } - } - } } } @@ -261,24 +252,78 @@ } } - // load offsets to term-document informations - for (InstantiatedDocument document : getDocumentsByNumber()) { - if (document == null) { - continue; // deleted - } - for (Fieldable field : document.getDocument().getFields()) { - if (field.isTermVectorStored() && field.isStoreOffsetWithTermVector()) { - TermPositionVector termPositionVector = (TermPositionVector) sourceIndexReader.getTermFreqVector(document.getDocumentNumber(), field.name()); - if (termPositionVector != null) { - for (int i = 0; i < termPositionVector.getTerms().length; i++) { - String token = termPositionVector.getTerms()[i]; - InstantiatedTerm term = findTerm(field.name(), token); - InstantiatedTermDocumentInformation termDocumentInformation = term.getAssociatedDocument(document.getDocumentNumber()); - termDocumentInformation.setTermOffsets(termPositionVector.getOffsets(i)); - } - } - } - } + int numberOfTVFields = termVecNames.size() + termVecOffsetNames.size() + termVecPosNames.size() + termVecPosOffNames.size(); + if(numberOfTVFields > 0) { + // load offsets to term-document informations if there is at least such field + for (InstantiatedDocument document : getDocumentsByNumber()) { + if (document == null) { + continue; // deleted + } + Map> vectorSpace = new HashMap>(numberOfTVFields, 1.0f); + for(String ivn : termVecNames) { + TermFreqVector termFreqVector = sourceIndexReader.getTermFreqVector(document.getDocumentNumber(), ivn); + if (termFreqVector != null) { + List al = new ArrayList(termFreqVector.getTerms().length); + vectorSpace.put(ivn, al); + for (int i = 0; i < termFreqVector.getTerms().length; i++) { + String token = termFreqVector.getTerms()[i]; + InstantiatedTerm term = findTerm(ivn, token); + InstantiatedTermDocumentInformation termDocumentInformation = term.getAssociatedDocument(document.getDocumentNumber()); + termDocumentInformation.setFreq(termFreqVector.getTermFrequencies()[i]); + al.add(termDocumentInformation); + } + } + } + + for(String ivn : termVecOffsetNames) { + TermPositionVector termPositionVector = (TermPositionVector) sourceIndexReader.getTermFreqVector(document.getDocumentNumber(), ivn); + if (termPositionVector != null) { + List al = new ArrayList(termPositionVector.getTerms().length); + vectorSpace.put(ivn, al); + for (int i = 0; i < termPositionVector.getTerms().length; i++) { + String token = termPositionVector.getTerms()[i]; + InstantiatedTerm term = findTerm(ivn, token); + InstantiatedTermDocumentInformation termDocumentInformation = term.getAssociatedDocument(document.getDocumentNumber()); + termDocumentInformation.setFreq(termPositionVector.getTermFrequencies()[i]); + termDocumentInformation.setTermOffsets(termPositionVector.getOffsets(i)); + al.add(termDocumentInformation); + } + } + } + + for(String ivn : termVecPosNames) { + TermPositionVector termPositionVector = (TermPositionVector) sourceIndexReader.getTermFreqVector(document.getDocumentNumber(), ivn); + if (termPositionVector != null) { + List al = new ArrayList(termPositionVector.getTerms().length); + vectorSpace.put(ivn, al); + for (int i = 0; i < termPositionVector.getTerms().length; i++) { + String token = termPositionVector.getTerms()[i]; + InstantiatedTerm term = findTerm(ivn, token); + InstantiatedTermDocumentInformation termDocumentInformation = term.getAssociatedDocument(document.getDocumentNumber()); + termDocumentInformation.setFreq(termPositionVector.getTermFrequencies()[i]); + termDocumentInformation.setTermPositions(termPositionVector.getTermPositions(i)); + al.add(termDocumentInformation); + } + } + } + for(String ivn : termVecPosOffNames) { + TermPositionVector termPositionVector = (TermPositionVector) sourceIndexReader.getTermFreqVector(document.getDocumentNumber(), ivn); + if (termPositionVector != null) { + List al = new ArrayList(termPositionVector.getTerms().length); + vectorSpace.put(ivn, al); + for (int i = 0; i < termPositionVector.getTerms().length; i++) { + String token = termPositionVector.getTerms()[i]; + InstantiatedTerm term = findTerm(ivn, token); + InstantiatedTermDocumentInformation termDocumentInformation = term.getAssociatedDocument(document.getDocumentNumber()); + termDocumentInformation.setFreq(termPositionVector.getTermFrequencies()[i]); + termDocumentInformation.setTermPositions(termPositionVector.getTermPositions(i)); + termDocumentInformation.setTermOffsets(termPositionVector.getOffsets(i)); + al.add(termDocumentInformation); + } + } + } + document.setVectorSpace(vectorSpace); + } } } Index: src/java/org/apache/lucene/store/instantiated/InstantiatedTermFreqVector.java =================================================================== --- src/java/org/apache/lucene/store/instantiated/InstantiatedTermFreqVector.java (revision 823249) +++ src/java/org/apache/lucene/store/instantiated/InstantiatedTermFreqVector.java (working copy) @@ -46,7 +46,7 @@ for (int i = 0; i < termDocumentInformations.size(); i++) { InstantiatedTermDocumentInformation termDocumentInformation = termDocumentInformations.get(i); terms[i] = termDocumentInformation.getTerm().text(); - termFrequencies[i] = termDocumentInformation.getTermPositions().length; + termFrequencies[i] = termDocumentInformation.getFreq(); } } Index: src/test/org/apache/lucene/store/instantiated/TestTermVector.java =================================================================== --- src/test/org/apache/lucene/store/instantiated/TestTermVector.java (revision 0) +++ src/test/org/apache/lucene/store/instantiated/TestTermVector.java (revision 0) @@ -0,0 +1,185 @@ +package org.apache.lucene.store.instantiated; +/** + * Copyright 2006 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.StringReader; +import java.util.HashSet; + +import junit.framework.TestCase; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.TermFreqVector; +import org.apache.lucene.index.TermPositionVector; +import org.apache.lucene.index.IndexWriter.MaxFieldLength; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.Version; + +/** + * Assert TermFreqVector is correctly injected in the instantiated index objects. + * + */ +public class TestTermVector extends TestCase { + public final static String fName = StringHelper.intern("content"); + + public void testTermFreqVector() throws CorruptIndexException, LockObtainFailedException, IOException { + InstantiatedIndex iid = getIndex(getDocument(TermVector.YES)); + InstantiatedIndexReader iir = iid.indexReaderFactory(); + TermFreqVector tfv = iir.getTermFreqVector(0, fName); + assertNotNull("TermFreqVector exists",tfv); + assertEquals("Number of terms",9, tfv.getTerms().length); + checkFreqs(tfv); + } + + public void testTermPosVector() throws CorruptIndexException, LockObtainFailedException, IOException { + InstantiatedIndex iid = getIndex(getDocument(TermVector.WITH_POSITIONS)); + InstantiatedIndexReader iir = iid.indexReaderFactory(); + TermPositionVector tfv = (TermPositionVector) iir.getTermFreqVector(0, fName); + assertNotNull("TermFreqVector exists",tfv); + assertEquals("Number of terms",9, tfv.getTerms().length); + checkFreqs(tfv); + checkPos(tfv); + } + + public void testTermOffVector() throws CorruptIndexException, LockObtainFailedException, IOException { + InstantiatedIndex iid = getIndex(getDocument(TermVector.WITH_OFFSETS)); + InstantiatedIndexReader iir = iid.indexReaderFactory(); + TermPositionVector tfv = (TermPositionVector) iir.getTermFreqVector(0, fName); + assertNotNull("TermFreqVector exists",tfv); + assertEquals("Number of terms",9, tfv.getTerms().length); + checkFreqs(tfv); + checkOffset(tfv); + } + + public void testTermVecOffVector() throws CorruptIndexException, LockObtainFailedException, IOException { + InstantiatedIndex iid = getIndex(getDocument(TermVector.WITH_POSITIONS_OFFSETS)); + InstantiatedIndexReader iir = iid.indexReaderFactory(); + TermPositionVector tfv = (TermPositionVector) iir.getTermFreqVector(0, fName); + assertNotNull("TermFreqVector exists",tfv); + assertEquals("Number of terms",9, tfv.getTerms().length); + checkFreqs(tfv); + checkPos(tfv); + checkOffset(tfv); + } + + public void testNoTermVector() throws CorruptIndexException, LockObtainFailedException, IOException { + InstantiatedIndex iid = getIndex(getDocument(TermVector.NO)); + InstantiatedIndexReader iir = iid.indexReaderFactory(); + TermPositionVector tfv = (TermPositionVector) iir.getTermFreqVector(0, fName); + assertNull("TermFreqVector does not exists",tfv); + } + + public Document getDocument(TermVector t) { + Document doc = new Document(); + Field f = new Field(fName, new StringReader("foobar is made of foo and bar where foo is preceding bar."), t); + doc.add(f); + return doc; + } + + public InstantiatedIndex getIndex(Document d) throws CorruptIndexException, LockObtainFailedException, IOException { + IndexWriter iw = new IndexWriter(new RAMDirectory(), new StandardAnalyzer(Version.LUCENE_CURRENT, new HashSet()),MaxFieldLength.UNLIMITED); + iw.addDocument(d); + return new InstantiatedIndex(iw.getReader()); + } + + public void checkFreqs(TermFreqVector tfv) { + boolean hasFoobar = false; + boolean hasBar = false; + boolean hasFoo = false; + for(int i = 0; i < tfv.getTerms().length; i++) { + String term = tfv.getTerms()[i]; + if(term.equals("foobar")) { + assertEquals("Freq for foobar", 1, tfv.getTermFrequencies()[i]); + hasFoobar = true; + } + if(term.equals("bar")) { + assertEquals("Freq for bar", 2, tfv.getTermFrequencies()[i]); + hasBar = true; + } + if(term.equals("foo")) { + assertEquals("Freq for foo", 2, tfv.getTermFrequencies()[i]); + hasFoo = true; + } + } + assertTrue("Has foobar", hasFoobar); + assertTrue("Has bar", hasBar); + assertTrue("Has foo", hasFoo); + } + + public void checkPos(TermPositionVector tfv) { + boolean hasFoobar = false; + boolean hasBar = false; + boolean hasFoo = false; + for(int i = 0; i < tfv.getTerms().length; i++) { + String term = tfv.getTerms()[i]; + if(term.equals("foobar")) { + assertEquals("Pos for foobar", 0, tfv.getTermPositions(i)[0]); + hasFoobar = true; + } + if(term.equals("bar")) { + assertEquals("Pos for bar", 6, tfv.getTermPositions(i)[0]); + assertEquals("Pos for bar", 11, tfv.getTermPositions(i)[1]); + hasBar = true; + } + if(term.equals("foo")) { + assertEquals("Pos for foo", 4, tfv.getTermPositions(i)[0]); + assertEquals("Pos for foo", 8, tfv.getTermPositions(i)[1]); + hasFoo = true; + } + } + assertTrue("Has foobar", hasFoobar); + assertTrue("Has bar", hasBar); + assertTrue("Has foo", hasFoo); + } + + public void checkOffset(TermPositionVector tfv) { + boolean hasFoobar = false; + boolean hasBar = false; + boolean hasFoo = false; + for(int i = 0; i < tfv.getTerms().length; i++) { + String term = tfv.getTerms()[i]; + if(term.equals("foobar")) { + assertEquals("Start Offset for foobar", 0, tfv.getOffsets(i)[0].getStartOffset()); + assertEquals("End Offset for foobar", 6, tfv.getOffsets(i)[0].getEndOffset()); + hasFoobar = true; + } + if(term.equals("bar")) { + assertEquals("Start Offset for bar", 26, tfv.getOffsets(i)[0].getStartOffset()); + assertEquals("End Offset for bar", 29, tfv.getOffsets(i)[0].getEndOffset()); + assertEquals("Start Offset for bar", 53, tfv.getOffsets(i)[1].getStartOffset()); + assertEquals("End Offset for bar", 56, tfv.getOffsets(i)[1].getEndOffset()); + hasBar = true; + } + if(term.equals("foo")) { + assertEquals("Start Offset for foo", 18, tfv.getOffsets(i)[0].getStartOffset()); + assertEquals("End Offset for foo", 21, tfv.getOffsets(i)[0].getEndOffset()); + assertEquals("Start Offset for foo", 36, tfv.getOffsets(i)[1].getStartOffset()); + assertEquals("End Offset for foo", 39, tfv.getOffsets(i)[1].getEndOffset()); + hasFoo = true; + } + } + assertTrue("Has foobar", hasFoobar); + assertTrue("Has bar", hasBar); + assertTrue("Has foo", hasFoo); + } +}