Index: src/java/org/apache/lucene/store/instantiated/FieldSetting.java =================================================================== --- src/java/org/apache/lucene/store/instantiated/FieldSetting.java (revision 0) +++ src/java/org/apache/lucene/store/instantiated/FieldSetting.java (revision 0) @@ -0,0 +1,42 @@ +package org.apache.lucene.store.instantiated; + +public class FieldSetting { + String fieldName; + + float boost = 1; + // private int dimensions = 0; // this is futuristic + int position = 0; + int offset; + int fieldLength = 0; + + boolean storeTermVector = false; + boolean storeOffsetWithTermVector = false; + boolean storePositionWithTermVector = false; + boolean omitNorms = false; + boolean isTokenized = false; + boolean storePayloads = false; + + boolean isStored = false; + boolean isIndexed = false; + boolean isBinary = false; + boolean isCompressed = false; + + // private float norm; + // private byte encodedNorm; + + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + final FieldSetting that = (FieldSetting) o; + + return fieldName.equals(that.fieldName); + + } + + public int hashCode() { + return fieldName.hashCode(); + } +} Index: src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java =================================================================== --- src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (revision 670653) +++ src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (working copy) @@ -16,15 +16,25 @@ * limitations under the License. */ +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.*; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermPositionVector; +import org.apache.lucene.index.TermPositions; -import java.io.IOException; -import java.io.Serializable; -import java.util.*; - /** * Represented as a coupled graph of class instances, this * all-in-memory index store implementation delivers search @@ -56,8 +66,8 @@ private InstantiatedTerm[] orderedTerms; private Map normsByFieldNameAndDocumentNumber; + Map fieldSettingsByFieldName; - /** * Creates an empty instantiated index for you to fill with data using an {@link org.apache.lucene.store.instantiated.InstantiatedIndexWriter}. */ @@ -68,12 +78,17 @@ void initialize() { // todo: clear index without loosing memory (uncouple stuff) termsByFieldAndText = new HashMap>(); + fieldSettingsByFieldName = new HashMap(); orderedTerms = new InstantiatedTerm[0]; documentsByNumber = new InstantiatedDocument[0]; normsByFieldNameAndDocumentNumber = new HashMap(); deletedDocuments = new HashSet(); } - + + public void mergeFieldSettings(Map map) { + this.fieldSettingsByFieldName.putAll(map); + } + /** * Creates a new instantiated index that looks just like the index in a specific state as represented by a reader. * @@ -83,7 +98,68 @@ public InstantiatedIndex(IndexReader sourceIndexReader) throws IOException { this(sourceIndexReader, null); } - + + private void initFieldSettings(IndexReader sourceIndexReader) { + Collection indexedNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.INDEXED); + for (String name : indexedNames) { + FieldSetting setting = getFieldSetting(name); + setting.isIndexed = true; + } + Collection indexedNoVecNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.INDEXED_NO_TERMVECTOR); + for (String name : indexedNoVecNames) { + FieldSetting setting = getFieldSetting(name); + setting.storeTermVector = false; + setting.isIndexed = true; + } + Collection indexedVecNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR); + for (String name : indexedVecNames) { + FieldSetting setting = getFieldSetting(name); + setting.storeTermVector = true; + setting.isIndexed = true; + } + Collection payloadNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.STORES_PAYLOADS); + for (String name : payloadNames) { + FieldSetting setting = getFieldSetting(name); + setting.storePayloads = true; + } + Collection termVecNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.TERMVECTOR); + for (String name : termVecNames) { + FieldSetting setting = getFieldSetting(name); + setting.storeTermVector = true; + } + Collection termVecOffsetNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET); + for (String name : termVecOffsetNames) { + FieldSetting setting = getFieldSetting(name); + setting.storeOffsetWithTermVector = true; + } + Collection termVecPosNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION); + for (String name : termVecPosNames) { + FieldSetting setting = getFieldSetting(name); + setting.storePositionWithTermVector = true; + } + Collection termVecPosOffNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET); + for (String name : termVecPosOffNames) { + FieldSetting setting = getFieldSetting(name); + setting.storeOffsetWithTermVector = true; + setting.storePositionWithTermVector = true; + } + Collection unindexedNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.UNINDEXED); + for (String name : unindexedNames) { + FieldSetting setting = getFieldSetting(name); + setting.isIndexed = false; + } + } + + private FieldSetting getFieldSetting(String name) { + FieldSetting fieldSetting = fieldSettingsByFieldName.get(name); + if (fieldSetting == null) { + fieldSetting = new FieldSetting(); + fieldSetting.fieldName = name; + fieldSettingsByFieldName.put(name, fieldSetting); + } + return fieldSetting; + } + /** * Creates a new instantiated index that looks just like the index in a specific state as represented by a reader. * @@ -98,7 +174,9 @@ } Collection allFieldNames = sourceIndexReader.getFieldNames(IndexReader.FieldOption.ALL); - + fieldSettingsByFieldName = new HashMap(); + initFieldSettings(sourceIndexReader); + initialize(); documentsByNumber = new InstantiatedDocument[sourceIndexReader.numDocs()]; Index: src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java =================================================================== --- src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (revision 670653) +++ src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (working copy) @@ -16,22 +16,37 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.index.*; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermFreqVector; +import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.TermVectorMapper; import org.apache.lucene.store.Directory; -import java.io.IOException; -import java.util.*; - /** - * An InstantiatedIndexReader is not a snapshot in time, - * it is completely in sync with the latest commit to the store! - * + * An InstantiatedIndexReader is not a snapshot in time, it is completely in + * sync with the latest commit to the store! + * * Consider using InstantiatedIndex as if it was immutable. */ -public class InstantiatedIndexReader - extends IndexReader { +public class InstantiatedIndexReader extends IndexReader { private final InstantiatedIndex index; @@ -47,40 +62,40 @@ return true; } - /** - * An InstantiatedIndexReader is not a snapshot in time, - * it is completely in sync with the latest commit to the store! - * - * @return output from {@link InstantiatedIndex#getVersion()} in associated instantiated index. + * An InstantiatedIndexReader is not a snapshot in time, it is completely in + * sync with the latest commit to the store! + * + * @return output from {@link InstantiatedIndex#getVersion()} in associated + * instantiated index. */ public long getVersion() { return index.getVersion(); } - public Directory directory() { throw new UnsupportedOperationException(); } - /** * An InstantiatedIndexReader is always current! - * - * Check whether this IndexReader is still using the - * current (i.e., most recently committed) version of the - * index. If a writer has committed any changes to the - * index since this reader was opened, this will return - * false, in which case you must open a new - * IndexReader in order to see the changes. See the - * description of the autoCommit - * flag which controls when the {@link IndexWriter} - * actually commits changes to the index. - * + * + * Check whether this IndexReader is still using the current (i.e., most + * recently committed) version of the index. If a writer has committed any + * changes to the index since this reader was opened, this will return + * false, in which case you must open a new IndexReader in + * order to see the changes. See the description of the autoCommit flag + * which controls when the {@link IndexWriter} actually commits changes to the + * index. + * * @return always true - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - * @throws UnsupportedOperationException unless overridden in subclass + * @throws CorruptIndexException + * if the index is corrupt + * @throws IOException + * if there is a low-level IO error + * @throws UnsupportedOperationException + * unless overridden in subclass */ public boolean isCurrent() throws IOException { return true; @@ -92,7 +107,7 @@ private Set deletedDocuments = new HashSet(); private Set deletedDocumentNumbers = new HashSet(); - private Map> updatedNormsByFieldNameAndDocumentNumber = null; + private Map> updatedNormsByFieldNameAndDocumentNumber = null; private class NormUpdate { private int doc; @@ -140,7 +155,7 @@ // 1. update norms if (updatedNormsByFieldNameAndDocumentNumber != null) { - for (Map.Entry> e : updatedNormsByFieldNameAndDocumentNumber.entrySet()) { + for (Map.Entry> e : updatedNormsByFieldNameAndDocumentNumber.entrySet()) { byte[] norms = getIndex().getNormsByFieldNameAndDocumentNumber().get(e.getKey()); for (NormUpdate normUpdate : e.getValue()) { norms[normUpdate.doc] = normUpdate.value; @@ -170,25 +185,56 @@ // ignored } - public Collection getFieldNames(FieldOption fldOption) { - if (fldOption != FieldOption.ALL) { - throw new IllegalArgumentException("Only FieldOption.ALL implemented."); // todo + public Collection getFieldNames(FieldOption fieldOption) { + Set fieldSet = new HashSet(); + for (FieldSetting fi : index.fieldSettingsByFieldName.values()) { + if (fieldOption == IndexReader.FieldOption.ALL) { + fieldSet.add(fi.fieldName); + } else if (!fi.isIndexed && fieldOption == IndexReader.FieldOption.UNINDEXED) { + fieldSet.add(fi.fieldName); + } else if (fi.storePayloads && fieldOption == IndexReader.FieldOption.STORES_PAYLOADS) { + fieldSet.add(fi.fieldName); + } else if (fi.isIndexed && fieldOption == IndexReader.FieldOption.INDEXED) { + fieldSet.add(fi.fieldName); + } else if (fi.isIndexed && fi.storeTermVector == false && fieldOption == IndexReader.FieldOption.INDEXED_NO_TERMVECTOR) { + fieldSet.add(fi.fieldName); + } else if (fi.storeTermVector == true && fi.storePositionWithTermVector == false && fi.storeOffsetWithTermVector == false + && fieldOption == IndexReader.FieldOption.TERMVECTOR) { + fieldSet.add(fi.fieldName); + } else if (fi.isIndexed && fi.storeTermVector && fieldOption == IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR) { + fieldSet.add(fi.fieldName); + } else if (fi.storePositionWithTermVector && fi.storeOffsetWithTermVector == false + && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION) { + fieldSet.add(fi.fieldName); + } else if (fi.storeOffsetWithTermVector && fi.storePositionWithTermVector == false + && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET) { + fieldSet.add(fi.fieldName); + } else if ((fi.storeOffsetWithTermVector && fi.storePositionWithTermVector) + && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET) { + fieldSet.add(fi.fieldName); + } } - return new ArrayList(getIndex().getTermsByFieldAndText().keySet()); + return fieldSet; } - /** - * This implementation ignores the field selector! All fields are always returned - * - * Get the {@link org.apache.lucene.document.Document} at the nth position. - * - * @param n Get the document at the nth position - * @param fieldSelector ignored - * @return The stored fields of the {@link org.apache.lucene.document.Document} at the nth position - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - * + * This implementation ignores the field selector! All fields are always + * returned + * + * Get the {@link org.apache.lucene.document.Document} at the nth + * position. + * + * @param n + * Get the document at the nth position + * @param fieldSelector + * ignored + * @return The stored fields of the + * {@link org.apache.lucene.document.Document} at the nth position + * @throws CorruptIndexException + * if the index is corrupt + * @throws IOException + * if there is a low-level IO error + * * @see org.apache.lucene.document.Fieldable * @see org.apache.lucene.document.FieldSelector * @see org.apache.lucene.document.SetBasedFieldSelector @@ -199,18 +245,17 @@ } public Document document(int n) throws IOException { - if ((deletedDocumentNumbers != null - && deletedDocumentNumbers.contains(n)) - || - (getIndex().getDeletedDocuments() != null - && getIndex().getDeletedDocuments().contains(n))) { - return null; - } + if (isDeleted(n)) return null; + //if ((deletedDocumentNumbers != null && isDeleted(n)//deletedDocumentNumbers.contains(n)) + // || (getIndex().getDeletedDocuments() != null && getIndex().getDeletedDocuments().contains(n))) { + // return null; + //} return getIndex().getDocumentsByNumber()[n].getDocument(); } /** - * never ever touch these values. it is the true values, unless norms have been touched. + * never ever touch these values. it is the true values, unless norms have + * been touched. */ public byte[] norms(String field) throws IOException { byte[] norms = getIndex().getNormsByFieldNameAndDocumentNumber().get(field); @@ -233,7 +278,8 @@ protected void doSetNorm(int doc, String field, byte value) throws IOException { if (updatedNormsByFieldNameAndDocumentNumber == null) { - updatedNormsByFieldNameAndDocumentNumber = new HashMap>(getIndex().getNormsByFieldNameAndDocumentNumber().size()); + updatedNormsByFieldNameAndDocumentNumber = new HashMap>(getIndex().getNormsByFieldNameAndDocumentNumber() + .size()); } List list = updatedNormsByFieldNameAndDocumentNumber.get(field); if (list == null) { @@ -252,7 +298,6 @@ } } - public TermEnum terms() throws IOException { return new InstantiatedTermEnum(this); } @@ -260,11 +305,11 @@ public TermEnum terms(Term t) throws IOException { InstantiatedTerm it = getIndex().findTerm(t); if (it != null) { - return new InstantiatedTermEnum(this, it.getTermIndex()); + return new InstantiatedTermEnum(this, it.getTermIndex()); } else { int startPos = Arrays.binarySearch(index.getOrderedTerms(), t, InstantiatedTerm.termComparator); if (startPos < 0) { - startPos = -1 -startPos; + startPos = -1 - startPos; } return new InstantiatedTermEnum(this, startPos); } @@ -293,19 +338,16 @@ public TermFreqVector getTermFreqVector(int docNumber, String field) throws IOException { InstantiatedDocument doc = getIndex().getDocumentsByNumber()[docNumber]; - if (doc.getVectorSpace() == null - || doc.getVectorSpace().get(field) == null) { + if (doc.getVectorSpace() == null || doc.getVectorSpace().get(field) == null) { return null; } else { return new InstantiatedTermPositionVector(doc, field); } } - public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) throws IOException { InstantiatedDocument doc = getIndex().getDocumentsByNumber()[docNumber]; - if (doc.getVectorSpace() != null - && doc.getVectorSpace().get(field) == null) { + if (doc.getVectorSpace() != null && doc.getVectorSpace().get(field) == null) { List tv = doc.getVectorSpace().get(field); mapper.setExpectations(field, tv.size(), true, true); for (InstantiatedTermDocumentInformation tdi : tv) { @@ -316,7 +358,7 @@ public void getTermFreqVector(int docNumber, TermVectorMapper mapper) throws IOException { InstantiatedDocument doc = getIndex().getDocumentsByNumber()[docNumber]; - for (Map.Entry> e : doc.getVectorSpace().entrySet()) { + for (Map.Entry> e : doc.getVectorSpace().entrySet()) { mapper.setExpectations(e.getKey(), e.getValue().size(), true, true); for (InstantiatedTermDocumentInformation tdi : e.getValue()) { mapper.map(tdi.getTerm().text(), tdi.getTermPositions().length, tdi.getTermOffsets(), tdi.getTermPositions()); Index: src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java =================================================================== --- src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (revision 670653) +++ src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (working copy) @@ -16,6 +16,22 @@ * limitations under the License. */ +import java.io.IOException; +import java.io.PrintStream; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; @@ -28,11 +44,6 @@ import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.Similarity; -import java.io.IOException; -import java.io.PrintStream; -import java.io.StringReader; -import java.util.*; - /** * This class, similar to {@link org.apache.lucene.index.IndexWriter}, has no locking mechanism. * @@ -161,6 +172,8 @@ boolean orderedTermsDirty = false; Set dirtyTerms = new HashSet(1000); + + Map fieldSettingsByFieldName = new HashMap(); InstantiatedDocument[] documentsByNumber = new InstantiatedDocument[index.getDocumentsByNumber().length + termDocumentInformationFactoryByDocument.size()]; System.arraycopy(index.getDocumentsByNumber(), 0, documentsByNumber, 0, index.getDocumentsByNumber().length); @@ -340,6 +353,7 @@ } } + fieldSettingsByFieldName.putAll(documentFieldSettingsByFieldName); } // order document informations in dirty terms @@ -357,7 +371,7 @@ // flush to writer index.setDocumentsByNumber(documentsByNumber); index.setOrderedTerms(orderedTerms.toArray(new InstantiatedTerm[orderedTerms.size()])); - + index.mergeFieldSettings(fieldSettingsByFieldName); // set term index if (orderedTermsDirty) { // todo optimize, only update from start position @@ -558,6 +572,7 @@ if (token.getPayload() != null && token.getPayload().length() > 0) { termDocumentInformationFactory.payloads.add(token.getPayload().toByteArray()); + fieldSettings.storePayloads = true; } else { termDocumentInformationFactory.payloads.add(null); } @@ -631,45 +646,6 @@ return analyzer; } - - private class FieldSetting { - private String fieldName; - - private float boost = 1; - //private int dimensions = 0; // this is futuristic - private int position = 0; - private int offset; - private int fieldLength = 0; - - private boolean storeTermVector = false; - private boolean storeOffsetWithTermVector = false; - private boolean storePositionWithTermVector = false; - private boolean omitNorms = false; - private boolean isTokenized = false; - - private boolean isStored = false; - private boolean isIndexed = false; - private boolean isBinary = false; - private boolean isCompressed = false; - - //private float norm; - //private byte encodedNorm; - - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - final FieldSetting that = (FieldSetting) o; - - return fieldName.equals(that.fieldName); - - } - - public int hashCode() { - return fieldName.hashCode(); - } - } - private class TermDocumentInformationFactory { private LinkedList payloads = new LinkedList(); private LinkedList termPositions = new LinkedList(); Index: src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocs.java =================================================================== --- src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocs.java (revision 670653) +++ src/java/org/apache/lucene/store/instantiated/InstantiatedTermDocs.java (working copy) @@ -121,16 +121,11 @@ } else { return true; } - - } /** * Does nothing */ public void close() { - } - - } Index: src/java/org/apache/lucene/store/instantiated/InstantiatedTermEnum.java =================================================================== --- src/java/org/apache/lucene/store/instantiated/InstantiatedTermEnum.java (revision 670653) +++ src/java/org/apache/lucene/store/instantiated/InstantiatedTermEnum.java (working copy) @@ -61,6 +61,7 @@ * Returns the current Term in the enumeration. */ public Term term() { + if (term == null) return null; return /*term == null ? null :*/ term.getTerm(); }