Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (date 1413355492000) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (revision ) @@ -174,4 +174,8 @@ public int getPropertyTypes() { return definition.getPropertyTypes(); } + + IndexDefinition getDefinition() { + return definition; + } } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (date 1413355492000) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (revision ) @@ -42,6 +42,7 @@ import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.FULL_TEXT_ENABLED; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_NAMES; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ORDERED_PROP_NAMES; public class IndexDefinition { private static final Logger log = LoggerFactory.getLogger(IndexDefinition.class); @@ -51,6 +52,8 @@ private final Set includes; + private final Set orderedProps; + private final boolean fullTextEnabled; private final boolean storageEnabled; @@ -79,6 +82,8 @@ this.excludes = toLowerCase(getMultiProperty(defn, EXCLUDE_PROPERTY_NAMES)); this.includes = getMultiProperty(defn, INCLUDE_PROPERTY_NAMES); this.fullTextEnabled = getOptionalValue(defn, FULL_TEXT_ENABLED, true); + this.orderedProps = getMultiProperty(defn, ORDERED_PROP_NAMES); + //Storage is disabled for non full text indexes this.storageEnabled = this.fullTextEnabled && getOptionalValue(defn, EXPERIMENTAL_STORAGE, true); @@ -88,6 +93,14 @@ propDefns.put(propName, new PropertyDefinition(this, propName, defn.child(propName))); } } + + // Add the types of the ordered but not indexed property names + for (String orderedProp : orderedProps) { + if (!propDefns.containsKey(orderedProp) && defn.hasChildNode(orderedProp)) { + propDefns.put(orderedProp, + new PropertyDefinition(this, orderedProp, defn.child(orderedProp))); + } + } this.propDefns = ImmutableMap.copyOf(propDefns); } @@ -103,6 +116,10 @@ return false; } return (propertyTypes & (1 << type)) != 0; + } + + boolean isOrdered(String name) { + return (!orderedProps.isEmpty() && orderedProps.contains(name)); } public NodeBuilder getDefinition() { Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (date 1413355492000) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (revision ) @@ -24,6 +24,7 @@ import javax.annotation.CheckForNull; +import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression; import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.lucene.index.IndexReader; @@ -134,7 +135,17 @@ && !o.getPropertyType().isArray()) { orderEntries.add(o); //Lucene can manage any order desc/asc } + if ((defn.includeProperty(o.getPropertyName()) || defn.isOrdered(o.getPropertyName())) + && o.getPropertyType() != null) { + // OrderEntry is immutable so, copy with type identified from the index definition + orderEntries.add(copyWithType(o, + Type.fromTag(defn.getPropDefn(o.getPropertyName()).getPropertyType(), + false))); //Lucene can manage any order desc/asc - } + } + } return orderEntries; + } + private static OrderEntry copyWithType(OrderEntry e, Type type) { + return new OrderEntry(e.getPropertyName(), type, e.getOrder()); } } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (date 1413355492000) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (revision ) @@ -79,6 +79,8 @@ import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TopDocs; @@ -334,16 +336,26 @@ try { IndexSearcher searcher = indexNode.getSearcher(); Query query = getQuery(filter, searcher.getIndexReader(), - nonFullTextConstraints, analyzer, indexNode.getDefinition()); + nonFullTextConstraints, analyzer, indexNode.getDefinition()); + Sort sort = getSort(plan.getSortOrder(), indexNode.getDefinition()); + TopDocs docs; long time = System.currentTimeMillis(); if (lastDoc != null) { LOG.debug("loading the next {} entries for query {}", nextBatchSize, query); + if (sort != null) { + docs = searcher.searchAfter(lastDoc, query, nextBatchSize, sort); + } else { - docs = searcher.searchAfter(lastDoc, query, nextBatchSize); + docs = searcher.searchAfter(lastDoc, query, nextBatchSize); + } } else { LOG.debug("loading the first {} entries for query {}", nextBatchSize, query); + if (sort != null) { + docs = searcher.search(query, nextBatchSize, sort); + } else { - docs = searcher.search(query, nextBatchSize); - } + docs = searcher.search(query, nextBatchSize); + } + } time = System.currentTimeMillis() - time; LOG.debug("... took {} ms", time); nextBatchSize = (int) Math.min(nextBatchSize * 2L, 100000); @@ -369,6 +381,37 @@ } }; return new LucenePathCursor(itr, settings); + } + + private static Sort getSort(List sortOrder, IndexDefinition definition) { + if (sortOrder == null || sortOrder.isEmpty()) { + return null; + } + SortField[] fields = new SortField[sortOrder.size()]; + for (int i = 0; i < sortOrder.size(); i++) { + OrderEntry oe = sortOrder.get(i); + boolean reverse = oe.getOrder() == OrderEntry.Order.ASCENDING ? false : true; + fields[i] = new SortField(oe.getPropertyName(), toLuceneSortType( + Type.fromTag(definition.getPropDefn(oe.getPropertyName()).getPropertyType(), + false)), reverse); + } + return new Sort(fields); + } + + private static SortField.Type toLuceneSortType(Type t) { + checkState(t != null, "Type cannot be null"); + checkState(!t.isArray(), "Array types are not supported"); + + switch (t.tag()) { + case PropertyType.LONG: + case PropertyType.DATE: + return SortField.Type.LONG; + case PropertyType.DOUBLE: + return SortField.Type.DOUBLE; + default: + //TODO Check about SortField.Type.STRING_VAL + return SortField.Type.STRING; + } } private static String getIndexName(IndexPlan plan){ Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (date 1413355492000) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (revision ) @@ -72,4 +72,9 @@ * contants for the possible values */ String PROP_TYPE = "propertyType"; + + /** + * Defines ordered properties for the index. + */ + String ORDERED_PROP_NAMES = "orderedProps"; } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (date 1413355492000) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (revision ) @@ -41,12 +41,16 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field; import org.apache.lucene.document.LongField; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.util.BytesRef; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.ParseContext; import org.apache.tika.sax.WriteOutContentHandler; @@ -233,7 +237,13 @@ } } } + + if (isVisible(pname) && !context.isFullTextEnabled() + && (context.getPropertyTypes() & (1 << property.getType().tag())) != 0 && context + .getDefinition().isOrdered(pname)) { + dirty = addTypedOrderedFields(fields, property); - } + } + } if (isUpdate && !dirty) { // updated the state but had no relevant changes @@ -277,6 +287,45 @@ f = new DoubleField(name, property.getValue(Type.DOUBLE, i), Field.Store.NO); } else if (tag == Type.BOOLEAN.tag()) { f = new StringField(name, property.getValue(Type.BOOLEAN, i).toString(), Field.Store.NO); + } + + if (f != null) { + this.context.indexUpdate(); + fields.add(f); + fieldAdded = true; + } + } + return fieldAdded; + } + + /** + * Add DocValueFields for declared ordered fields + * + * @param fields + * @param property + * @return + * @throws CommitFailedException + */ + private boolean addTypedOrderedFields(List fields, PropertyState property) throws + CommitFailedException { + int tag = property.getType().tag(); + String name = property.getName(); + boolean fieldAdded = false; + for (int i = 0; i < property.count(); i++) { + Field f = null; + if (tag == Type.LONG.tag()) { + f = new NumericDocValuesField(name, property.getValue(Type.LONG, i)); + } else if (tag == Type.DATE.tag()) { + f = new NumericDocValuesField(name, + FieldFactory.dateToLong(property.getValue(Type.DATE, i))); + } else if (tag == Type.DOUBLE.tag()) { + f = new DoubleDocValuesField(name, property.getValue(Type.DOUBLE, i)); + } else if (tag == Type.BOOLEAN.tag()) { + f = new SortedDocValuesField(name, + new BytesRef(property.getValue(Type.BOOLEAN, i).toString())); + } else if (tag == Type.STRING.tag()) { + f = new SortedDocValuesField(name, + new BytesRef(property.getValue(Type.STRING, i).toString())); } if (f != null) {