Index: core/src/java/org/apache/lucene/analysis/Analyzer.java =================================================================== --- core/src/java/org/apache/lucene/analysis/Analyzer.java (revision 1495776) +++ core/src/java/org/apache/lucene/analysis/Analyzer.java (arbetskopia) @@ -163,6 +163,7 @@ * including across IndexableField instances, are in successive positions, allowing * exact PhraseQuery matches, for instance, across IndexableField instance boundaries. * + * @see org.apache.lucene.index.IndexableField#positionIncrementGap() * @param fieldName IndexableField name being indexed. * @return position increment gap, added to the next token emitted from {@link #tokenStream(String,Reader)}. * This value must be {@code >= 0}. @@ -177,6 +178,7 @@ * This method is only called if the field * produced at least one token for indexing. * + * @see org.apache.lucene.index.IndexableField#offsetGap() * @param fieldName the field just indexed * @return offset gap, added to the next token emitted from {@link #tokenStream(String,Reader)}. * This value must be {@code >= 0}. Index: core/src/java/org/apache/lucene/document/Field.java =================================================================== --- core/src/java/org/apache/lucene/document/Field.java (revision 1495776) +++ core/src/java/org/apache/lucene/document/Field.java (arbetskopia) @@ -74,6 +74,10 @@ * customize how it's tokenized */ protected TokenStream tokenStream; + protected int positionIncrementGap = 0; + + protected int offsetGap = 0; + private transient TokenStream internalTokenStream; private transient ReusableStringReader internalReader; @@ -561,7 +565,25 @@ throw new IllegalArgumentException("Field must have either TokenStream, String, Reader or Number value"); } - + + @Override + public int positionIncrementGap() { + return positionIncrementGap; + } + + @Override + public int offsetGap() { + return offsetGap; + } + + public void setPositionIncrementGap(int positionIncrementGap) { + this.positionIncrementGap = positionIncrementGap; + } + + public void setOffsetGap(int offsetGap) { + this.offsetGap = offsetGap; + } + static final class ReusableStringReader extends Reader { private int pos = 0, size = 0; private String s = null; Index: core/src/java/org/apache/lucene/index/DocInverterPerField.java =================================================================== --- core/src/java/org/apache/lucene/index/DocInverterPerField.java (revision 1495776) +++ core/src/java/org/apache/lucene/index/DocInverterPerField.java (arbetskopia) @@ -89,7 +89,7 @@ int lastStartOffset = 0; if (i > 0) { - fieldState.position += analyzed ? docState.analyzer.getPositionIncrementGap(fieldInfo.name) : 0; + fieldState.position += analyzed ? docState.analyzer.getPositionIncrementGap(fieldInfo.name) + field.positionIncrementGap() : 0; } final TokenStream stream = field.tokenStream(docState.analyzer); @@ -186,7 +186,7 @@ } } - fieldState.offset += analyzed ? docState.analyzer.getOffsetGap(fieldInfo.name) : 0; + fieldState.offset += analyzed ? docState.analyzer.getOffsetGap(fieldInfo.name) + field.offsetGap() : 0; fieldState.boost *= field.boost(); } Index: core/src/java/org/apache/lucene/index/IndexableField.java =================================================================== --- core/src/java/org/apache/lucene/index/IndexableField.java (revision 1495776) +++ core/src/java/org/apache/lucene/index/IndexableField.java (arbetskopia) @@ -69,4 +69,32 @@ * @see DefaultSimilarity#encodeNormValue(float) */ public float boost(); + + /** + * Invoked before indexing a IndexableField instance if + * terms have already been added to that field. This allows + * to place an automatic position increment gap between + * IndexbleField instances. The default value + * position increment gap is 0. With a 0 position increment gap and + * the typical default token position increment of 1, all terms in a field, + * including across IndexableField instances, are in successive positions, allowing + * exact PhraseQuery matches, for instance, across IndexableField instance boundaries. + * + * @see Analyzer#getPositionIncrementGap(String) + * @return position increment gap, this value must be {@code >= 0}. + */ + public int positionIncrementGap(); + + /** + * Just like {@link #positionIncrementGap}, except for + * Token offsets instead. By default this returns 1. + * This method is only called if the field + * produced at least one token for indexing. + * + * @see Analyzer#getOffsetGap(String) + * @return offset gap, this value must be {@code >= 0}. + */ + public int offsetGap(); + + } Index: core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1495776) +++ core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (arbetskopia) @@ -1599,6 +1599,16 @@ public TokenStream tokenStream(Analyzer analyzer) throws IOException { return null; } + + @Override + public int positionIncrementGap() { + return 0; + } + + @Override + public int offsetGap() { + return 0; + } }); } return list; Index: core/src/test/org/apache/lucene/index/TestIndexableField.java =================================================================== --- core/src/test/org/apache/lucene/index/TestIndexableField.java (revision 1495776) +++ core/src/test/org/apache/lucene/index/TestIndexableField.java (arbetskopia) @@ -43,6 +43,7 @@ private final int counter; private final IndexableFieldType fieldType = new IndexableFieldType() { + @Override public boolean indexed() { return (counter % 10) != 3; @@ -92,6 +93,8 @@ public DocValuesType docValueType() { return null; } + + }; public MyField(int counter) { @@ -155,6 +158,16 @@ return readerValue() != null ? analyzer.tokenStream(name(), readerValue()) : analyzer.tokenStream(name(), new StringReader(stringValue())); } + + @Override + public int positionIncrementGap() { + return 0; //todo some test, assert gap. + } + + @Override + public int offsetGap() { + return 0; //todo some test, assert gap. + } } // Silly test showing how to index documents w/o using Lucene's core