Index: lucene/src/java/org/apache/lucene/index/DocInverterPerField.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (revision d9865d425800ad499ea272c67c720bd473699e18) +++ lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (revision ) @@ -138,8 +138,8 @@ PositionIncrementAttribute posIncrAttribute = fieldState.attributeSource.addAttribute(PositionIncrementAttribute.class); consumer.start(field); - + - for(;;) { + for(boolean firstToken = true; hasMoreTokens; firstToken=false) { // If we hit an exception in stream.next below // (which is fairly common, eg if analyzer @@ -147,16 +147,15 @@ // non-aborting and (above) this one document // will be marked as deleted, but still // consume a docID - + - if (!hasMoreTokens) break; - final int posIncr = posIncrAttribute.getPositionIncrement(); fieldState.position += posIncr; - if (fieldState.position > 0) { - fieldState.position--; - } + if (firstToken && i == 0)//i.e. this is the very first token we emit for this field in this document + fieldState.position--;//we want to start at 0, not 1 + if (fieldState.position < 0)//abnormal + fieldState.position = 0;//"LUCENE-1542, pos < 0 doesn't work with payloads"; - if (posIncr == 0) + if (posIncr == 0 && !firstToken) fieldState.numOverlap++; boolean success = false; @@ -173,7 +172,6 @@ if (!success) docState.docWriter.setAborting(); } - fieldState.position++; if (++fieldState.length >= maxFieldLength) { if (docState.infoStream != null) docState.infoStream.println("maxFieldLength " +maxFieldLength+ " reached for field " + fieldInfo.name + ", ignoring following tokens"); Index: lucene/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java (revision ) +++ lucene/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java (revision ) @@ -0,0 +1,67 @@ +package org.apache.lucene.search.spans; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.RegExp; + +public class TestSpanFirstQuery extends LuceneTestCase { + public void testStartPositions() throws Exception { + Directory dir = newDirectory(); + + // mimic StopAnalyzer + CharacterRunAutomaton stopSet = new CharacterRunAutomaton(new RegExp("the|a|of").toAutomaton()); + Analyzer analyzer = new MockAnalyzer(MockTokenizer.SIMPLE, true, stopSet, true); + + RandomIndexWriter writer = new RandomIndexWriter(random, dir, analyzer); + Document doc = new Document(); + doc.add(newField("field", "the quick brown fox", Field.Index.ANALYZED)); + writer.addDocument(doc); + Document doc2 = new Document(); + doc2.add(newField("field", "quick brown fox", Field.Index.ANALYZED)); + writer.addDocument(doc2); + + IndexReader reader = writer.getReader(); + IndexSearcher searcher = new IndexSearcher(reader); + + // user queries on "starts-with quick" + SpanQuery sfq = new SpanFirstQuery(new SpanTermQuery(new Term("field", "quick")), 1); + assertEquals(1, searcher.search(sfq, 10).totalHits); + + // user queries on "starts-with the quick" + SpanQuery include = new SpanFirstQuery(new SpanTermQuery(new Term("field", "quick")), 2); + sfq = new SpanNotQuery(include, sfq); + assertEquals(1, searcher.search(sfq, 10).totalHits); + + writer.close(); + searcher.close(); + reader.close(); + dir.close(); + } +} Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision d9865d425800ad499ea272c67c720bd473699e18) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision ) @@ -4311,28 +4311,50 @@ // LUCENE-2529 public void testPositionIncrementGapEmptyField() throws Exception { Directory dir = newDirectory(); - MockAnalyzer analyzer = new MockAnalyzer(); + MockAnalyzer analyzer = new MockAnalyzer(MockTokenizer.WHITESPACE,true,MockTokenFilter.ENGLISH_STOPSET,true); - analyzer.setPositionIncrementGap( 100 ); + analyzer.setPositionIncrementGap(100); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - Field f = newField("field", "", Field.Store.NO, - Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS); - Field f2 = newField("field", "crunch man", Field.Store.NO, - Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS); - doc.add(f); - doc.add(f2); + doc.add(newField("fieldNotAnalyzed", "category", + Store.NO, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS)); + + doc.add(newField("field", "", + Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS)); + doc.add(newField("field", "crunch man", + Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS)); + doc.add(newField("field", "", + Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS)); + doc.add(newField("field", "z", + Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS)); + + doc.add(newField("field2", "the zoo",//starts with stop word + Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS)); w.addDocument(doc); w.close(); IndexReader r = IndexReader.open(dir, true); - TermPositionVector tpv = ((TermPositionVector) r.getTermFreqVector(0, "field")); - int[] poss = tpv.getTermPositions(0); + TermPositionVector tpv = ((TermPositionVector) r.getTermFreqVector(0, "fieldNotAnalyzed")); + int[] poss = tpv.getTermPositions(0);//category assertEquals(1, poss.length); - assertEquals(100, poss[0]); - poss = tpv.getTermPositions(1); + assertEquals(0, poss[0]); + + tpv = ((TermPositionVector) r.getTermFreqVector(0, "field")); + poss = tpv.getTermPositions(0);//crunch assertEquals(1, poss.length); assertEquals(101, poss[0]); + poss = tpv.getTermPositions(1);//man + assertEquals(1, poss.length); + assertEquals(102, poss[0]); + poss = tpv.getTermPositions(2);//z + assertEquals(1, poss.length); + assertEquals(303, poss[0]); + + tpv = ((TermPositionVector) r.getTermFreqVector(0, "field2")); + poss = tpv.getTermPositions(0);//zoo + assertEquals(1, poss.length); + assertEquals(1, poss[0]); + r.close(); dir.close(); }