Note that for simple usage, {@link NumericField} is - * recommended. {@link NumericField} disables norms and - * term freqs, as they are not usually needed during - * searching. If you need to change these settings, you - * should use this class. - * - *
See {@link NumericField} for capabilities of fields - * indexed numerically.
- * - *Here's an example usage, for an int field:
- *
- *
- * Field field = new Field(name, new NumericTokenStream(precisionStep).setIntValue(value)); - * field.setOmitNorms(true); - * field.setOmitTermFreqAndPositions(true); - * document.add(field); - *- * - *
For optimal performance, re-use the TokenStream and Field instance - * for more than one document: - * - *
- * NumericTokenStream stream = new NumericTokenStream(precisionStep);
- * Field field = new Field(name, stream);
- * field.setOmitNorms(true);
- * field.setOmitTermFreqAndPositions(true);
- * Document document = new Document();
- * document.add(field);
- *
- * for(all documents) {
- * stream.setIntValue(value)
- * writer.addDocument(document);
- * }
- *
- *
- * This stream is not intended to be used in analyzers; - * it's more for iterating the different precisions during - * indexing a specific numeric value.
- - *NOTE: as token streams are only consumed once
- * the document is added to the index, if you index more
- * than one numeric field, use a separate NumericTokenStream
- * instance for each.
See {@link NumericRangeQuery} for more details on the
- * precisionStep
- * parameter as well as how numeric fields work under the hood.
precisionStep
- * {@link NumericUtils#PRECISION_STEP_DEFAULT} (4). The stream is not yet initialized,
- * before using set a value using the various set???Value() methods.
- */
- public NumericTokenStream() {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, NumericUtils.PRECISION_STEP_DEFAULT);
- }
-
- /**
- * Creates a token stream for numeric values with the specified
- * precisionStep. The stream is not yet initialized,
- * before using set a value using the various set???Value() methods.
- */
- public NumericTokenStream(final int precisionStep) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, precisionStep);
- }
-
- /**
- * Expert: Creates a token stream for numeric values with the specified
- * precisionStep using the given
- * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
- * The stream is not yet initialized,
- * before using set a value using the various set???Value() methods.
- */
- public NumericTokenStream(AttributeFactory factory, final int precisionStep) {
- super(new NumericAttributeFactory(factory));
- if (precisionStep < 1)
- throw new IllegalArgumentException("precisionStep must be >=1");
- this.precisionStep = precisionStep;
- numericAtt.setShift(-precisionStep);
- }
-
- /**
- * Initializes the token stream with the supplied long value.
- * @param value the value, for which this TokenStream should enumerate tokens.
- * @return this instance, because of this you can use it the following way:
- * new Field(name, new NumericTokenStream(precisionStep).setLongValue(value))
- */
- public NumericTokenStream setLongValue(final long value) {
- numericAtt.init(value, valSize = 64, precisionStep, -precisionStep);
- return this;
- }
-
- /**
- * Initializes the token stream with the supplied int value.
- * @param value the value, for which this TokenStream should enumerate tokens.
- * @return this instance, because of this you can use it the following way:
- * new Field(name, new NumericTokenStream(precisionStep).setIntValue(value))
- */
- public NumericTokenStream setIntValue(final int value) {
- numericAtt.init(value, valSize = 32, precisionStep, -precisionStep);
- return this;
- }
-
- /**
- * Initializes the token stream with the supplied double value.
- * @param value the value, for which this TokenStream should enumerate tokens.
- * @return this instance, because of this you can use it the following way:
- * new Field(name, new NumericTokenStream(precisionStep).setDoubleValue(value))
- */
- public NumericTokenStream setDoubleValue(final double value) {
- numericAtt.init(NumericUtils.doubleToSortableLong(value), valSize = 64, precisionStep, -precisionStep);
- return this;
- }
-
- /**
- * Initializes the token stream with the supplied float value.
- * @param value the value, for which this TokenStream should enumerate tokens.
- * @return this instance, because of this you can use it the following way:
- * new Field(name, new NumericTokenStream(precisionStep).setFloatValue(value))
- */
- public NumericTokenStream setFloatValue(final float value) {
- numericAtt.init(NumericUtils.floatToSortableInt(value), valSize = 32, precisionStep, -precisionStep);
- return this;
- }
-
- @Override
- public void reset() {
- if (valSize == 0)
- throw new IllegalStateException("call set???Value() before usage");
- numericAtt.setShift(-precisionStep);
- }
-
- @Override
- public boolean incrementToken() {
- if (valSize == 0)
- throw new IllegalStateException("call set???Value() before usage");
-
- // this will only clear all other attributes in this TokenStream
- clearAttributes();
-
- final int shift = numericAtt.incShift();
- typeAtt.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC : TOKEN_TYPE_LOWER_PREC);
- posIncrAtt.setPositionIncrement((shift == 0) ? 1 : 0);
- return (shift < valSize);
- }
-
- /** Returns the precision step. */
- public int getPrecisionStep() {
- return precisionStep;
- }
-
- // members
- private final NumericTermAttribute numericAtt = addAttribute(NumericTermAttribute.class);
- private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
- private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-
- private int valSize = 0; // valSize==0 means not initialized
- private final int precisionStep;
-}
Index: lucene/src/java/org/apache/lucene/analysis/CharReader.java
===================================================================
--- lucene/src/java/org/apache/lucene/analysis/CharReader.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/analysis/CharReader.java (working copy)
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis;
-
-import java.io.IOException;
-import java.io.Reader;
-
-/**
- * CharReader is a Reader wrapper. It reads chars from
- * Reader and outputs {@link CharStream}, defining an
- * identify function {@link #correctOffset} method that
- * simply returns the provided offset.
- */
-public final class CharReader extends CharStream {
-
- private final Reader input;
-
- public static CharStream get(Reader input) {
- return input instanceof CharStream ?
- (CharStream)input : new CharReader(input);
- }
-
- private CharReader(Reader in) {
- input = in;
- }
-
- @Override
- public int correctOffset(int currentOff) {
- return currentOff;
- }
-
- @Override
- public void close() throws IOException {
- input.close();
- }
-
- @Override
- public int read(char[] cbuf, int off, int len) throws IOException {
- return input.read(cbuf, off, len);
- }
-
- @Override
- public boolean markSupported(){
- return input.markSupported();
- }
-
- @Override
- public void mark( int readAheadLimit ) throws IOException {
- input.mark(readAheadLimit);
- }
-
- @Override
- public void reset() throws IOException {
- input.reset();
- }
-}
Index: lucene/src/java/org/apache/lucene/analysis/Token.java
===================================================================
--- lucene/src/java/org/apache/lucene/analysis/Token.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/analysis/Token.java (working copy)
@@ -1,647 +0,0 @@
-package org.apache.lucene.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.index.Payload;
-import org.apache.lucene.index.DocsAndPositionsEnum; // for javadoc
-import org.apache.lucene.util.Attribute;
-import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.AttributeImpl;
-import org.apache.lucene.util.AttributeReflector;
-
-/**
- A Token is an occurrence of a term from the text of a field. It consists of
- a term's text, the start and end offset of the term in the text of the field,
- and a type string.
- - The start and end offsets permit applications to re-associate a token with - its source text, e.g., to display highlighted query terms in a document - browser, or to show matching text fragments in a KWIC - display, etc. -
- The type is a string, assigned by a lexical analyzer - (a.k.a. tokenizer), naming the lexical or syntactic class that the token - belongs to. For example an end of sentence marker token might be implemented - with type "eos". The default token type is "word". -
- A Token can optionally have metadata (a.k.a. Payload) in the form of a variable
- length byte array. Use {@link DocsAndPositionsEnum#getPayload()} to retrieve the
- payloads from the index.
-
-
-
-
NOTE: As of 2.9, Token implements all {@link Attribute} interfaces
- that are part of core Lucene and can be found in the {@code tokenattributes} subpackage.
- Even though it is not necessary to use Token anymore, with the new TokenStream API it can
- be used as convenience class that implements all {@link Attribute}s, which is especially useful
- to easily switch from the old to the new TokenStream API.
-
-
-
-
Tokenizers and TokenFilters should try to re-use a Token - instance when possible for best performance, by - implementing the {@link TokenStream#incrementToken()} API. - Failing that, to create a new Token you should first use - one of the constructors that starts with null text. To load - the token from a char[] use {@link #copyBuffer(char[], int, int)}. - To load from a String use {@link #setEmpty} followed by {@link #append(CharSequence)} or {@link #append(CharSequence, int, int)}. - Alternatively you can get the Token's termBuffer by calling either {@link #buffer()}, - if you know that your text is shorter than the capacity of the termBuffer - or {@link #resizeBuffer(int)}, if there is any possibility - that you may need to grow the buffer. Fill in the characters of your term into this - buffer, with {@link String#getChars(int, int, char[], int)} if loading from a string, - or with {@link System#arraycopy(Object, int, Object, int, int)}, and finally call {@link #setLength(int)} to - set the length of the term text. See LUCENE-969 - for details.
-Typical Token reuse patterns: -
- return reusableToken.reinit(string, startOffset, endOffset[, type]); --
- return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]); --
- return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]); --
- return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]); --
- return reusableToken.reinit(source.buffer(), 0, source.length(), source.startOffset(), source.endOffset()[, source.type()]); --
TokenStreams can be chained, one cannot assume that the Token's current type is correct.
- Please note: With Lucene 3.1, the {@linkplain #toString toString()} method had to be changed to match the
- {@link CharSequence} interface introduced by the interface {@link org.apache.lucene.analysis.tokenattributes.CharTermAttribute}.
- This method now only prints the term text, no additional information anymore.
-
The default value is one. - * - *
Some common uses for this are:
Token as implementation for the basic
- * attributes and return the default impl (with "Impl" appended) for all other
- * attributes.
- * @since 3.0
- */
- public static final AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
- new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
-
- /** Expert: Creates a TokenAttributeFactory returning {@link Token} as instance for the basic attributes
- * and for all other attributes calls the given delegate factory.
- * @since 3.0
- */
- public static final class TokenAttributeFactory extends AttributeSource.AttributeFactory {
-
- private final AttributeSource.AttributeFactory delegate;
-
- /** Expert: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
- * and for all other attributes calls the given delegate factory. */
- public TokenAttributeFactory(AttributeSource.AttributeFactory delegate) {
- this.delegate = delegate;
- }
-
- @Override
- public AttributeImpl createAttributeInstance(Class attClass) {
- return attClass.isAssignableFrom(Token.class)
- ? new Token() : delegate.createAttributeInstance(attClass);
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) return true;
- if (other instanceof TokenAttributeFactory) {
- final TokenAttributeFactory af = (TokenAttributeFactory) other;
- return this.delegate.equals(af.delegate);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return delegate.hashCode() ^ 0x0a45aa31;
- }
- }
-
-}
Index: lucene/src/java/org/apache/lucene/analysis/CachingTokenFilter.java
===================================================================
--- lucene/src/java/org/apache/lucene/analysis/CachingTokenFilter.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/analysis/CachingTokenFilter.java (working copy)
@@ -1,86 +0,0 @@
-package org.apache.lucene.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.lucene.util.AttributeSource;
-
-/**
- * This class can be used if the token attributes of a TokenStream
- * are intended to be consumed more than once. It caches
- * all token attribute states locally in a List.
- *
- * CachingTokenFilter implements the optional method
- * {@link TokenStream#reset()}, which repositions the
- * stream to the first Token.
- */
-public final class CachingTokenFilter extends TokenFilter {
- private List
- * Typical implementations first build a Tokenizer, which breaks the stream of
- * characters from the Reader into raw Tokens. One or more TokenFilters may
- * then be applied to the output of the Tokenizer.
- * The {@code Analyzer}-API in Lucene is based on the decorator pattern.
- * Therefore all non-abstract subclasses must be final or their {@link #tokenStream}
- * and {@link #reusableTokenStream} implementations must be final! This is checked
- * when Java assertions are enabled.
- */
-public abstract class Analyzer implements Closeable {
-
- protected Analyzer() {
- super();
- assert assertFinal();
- }
-
- private boolean assertFinal() {
- try {
- final Class clazz = getClass();
- assert clazz.isAnonymousClass() ||
- (clazz.getModifiers() & (Modifier.FINAL | Modifier.PRIVATE)) != 0 ||
- (
- Modifier.isFinal(clazz.getMethod("tokenStream", String.class, Reader.class).getModifiers()) &&
- Modifier.isFinal(clazz.getMethod("reusableTokenStream", String.class, Reader.class).getModifiers())
- ) :
- "Analyzer implementation classes or at least their tokenStream() and reusableTokenStream() implementations must be final";
- return true;
- } catch (NoSuchMethodException nsme) {
- return false;
- }
- }
-
- /** Creates a TokenStream which tokenizes all the text in the provided
- * Reader. Must be able to handle null field name for
- * backward compatibility.
- */
- public abstract TokenStream tokenStream(String fieldName, Reader reader);
-
- /** Creates a TokenStream that is allowed to be re-used
- * from the previous time that the same thread called
- * this method. Callers that do not need to use more
- * than one TokenStream at the same time from this
- * analyzer should use this method for better
- * performance.
- */
- public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
- return tokenStream(fieldName, reader);
- }
-
- private CloseableThreadLocal
- The synergy between {@link org.apache.lucene.analysis.Analyzer} and {@link org.apache.lucene.analysis.Tokenizer}
- is sometimes confusing. To ease on this confusion, some clarifications:
- Hints, Tips and Traps
-
-
-
- Lucene Java provides a number of analysis capabilities, the most commonly used one being the StandardAnalyzer. - Many applications will have a long and industrious life with nothing more - than the StandardAnalyzer. However, there are a few other classes/packages that are worth mentioning: -
- Analysis is one of the main causes of performance degradation during indexing. Simply put, the more you analyze the slower the indexing (in most cases). - Perhaps your application would be just fine using the simple WhitespaceTokenizer combined with a StopFilter. The contrib/benchmark library can be useful - for testing out the speed of the analysis process. -
-- Applications usually do not invoke analysis – Lucene does it for them: -
- Analyzer analyzer = new StandardAnalyzer(); // or any other analyzer
- TokenStream ts = analyzer.tokenStream("myfield",new StringReader("some text goes here"));
- while (ts.incrementToken()) {
- System.out.println("token: "+ts));
- }
-
-
-- Selecting the "correct" analyzer is crucial - for search quality, and can also affect indexing and search performance. - The "correct" analyzer differs between applications. - Lucene java's wiki page - AnalysisParalysis - provides some data on "analyzing your analyzer". - Here are some rules of thumb: -
Creating your own Analyzer is straightforward. It usually involves either wrapping an existing Tokenizer and set of TokenFilters to create a new Analyzer -or creating both the Analyzer and a Tokenizer or TokenFilter. Before pursuing this approach, you may find it worthwhile -to explore the modules/analysis library and/or ask on the java-user@lucene.apache.org mailing list first to see if what you need already exists. -If you are still committed to creating your own Analyzer or TokenStream derivation (Tokenizer or TokenFilter) have a look at -the source code of any one of the many samples located in this package. -
-- The following sections discuss some aspects of implementing your own analyzer. -
-- When {@link org.apache.lucene.document.Document#add(org.apache.lucene.document.Fieldable) document.add(field)} - is called multiple times for the same field name, we could say that each such call creates a new - section for that field in that document. - In fact, a separate call to - {@link org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String, java.io.Reader) tokenStream(field,reader)} - would take place for each of these so called "sections". - However, the default Analyzer behavior is to treat all these sections as one large section. - This allows phrase search and proximity search to seamlessly cross - boundaries between these "sections". - In other words, if a certain field "f" is added like this: -
- document.add(new Field("f","first ends",...);
- document.add(new Field("f","starts two",...);
- indexWriter.addDocument(document);
-
- Then, a phrase search for "ends starts" would find that document.
- Where desired, this behavior can be modified by introducing a "position gap" between consecutive field "sections",
- simply by overriding
- {@link org.apache.lucene.analysis.Analyzer#getPositionIncrementGap(java.lang.String) Analyzer.getPositionIncrementGap(fieldName)}:
-
- Analyzer myAnalyzer = new StandardAnalyzer() {
- public int getPositionIncrementGap(String fieldName) {
- return 10;
- }
- };
-
-
-- By default, all tokens created by Analyzers and Tokenizers have a - {@link org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute#getPositionIncrement() position increment} of one. - This means that the position stored for that token in the index would be one more than - that of the previous token. - Recall that phrase and proximity searches rely on position info. -
-- If the selected analyzer filters the stop words "is" and "the", then for a document - containing the string "blue is the sky", only the tokens "blue", "sky" are indexed, - with position("sky") = 1 + position("blue"). Now, a phrase query "blue is the sky" - would find that document, because the same analyzer filters the same stop words from - that query. But also the phrase query "blue sky" would find that document. -
-- If this behavior does not fit the application needs, - a modified analyzer can be used, that would increment further the positions of - tokens following a removed stop word, using - {@link org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute#setPositionIncrement(int)}. - This can be done with something like: -
- public TokenStream tokenStream(final String fieldName, Reader reader) {
- final TokenStream ts = someAnalyzer.tokenStream(fieldName, reader);
- TokenStream res = new TokenStream() {
- CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-
- public boolean incrementToken() throws IOException {
- int extraIncrement = 0;
- while (true) {
- boolean hasNext = ts.incrementToken();
- if (hasNext) {
- if (stopWords.contains(termAtt.toString())) {
- extraIncrement++; // filter this word
- continue;
- }
- if (extraIncrement>0) {
- posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement()+extraIncrement);
- }
- }
- return hasNext;
- }
- }
- };
- return res;
- }
-
- Now, with this modified analyzer, the phrase query "blue sky" would find that document.
- But note that this is yet not a perfect solution, because any phrase query "blue w1 w2 sky"
- where both w1 and w2 are stop words would match that document.
-
-- Few more use cases for modifying position increments are: -
- With Lucene 2.9 we introduce a new TokenStream API. The old API used to produce Tokens. A Token - has getter and setter methods for different properties like positionIncrement and termText. - While this approach was sufficient for the default indexing format, it is not versatile enough for - Flexible Indexing, a term which summarizes the effort of making the Lucene indexer pluggable and extensible for custom - index formats. -
--A fully customizable indexer means that users will be able to store custom data structures on disk. Therefore an API -is necessary that can transport custom types of data from the documents to the indexer. -
-- Lucene now provides six Attributes out of the box, which replace the variables the Token class has: -
The term text of a token.
The start and end offset of token in characters.
See above for detailed information about position increment.
The payload that a Token can optionally have.
The type of the token. Default is 'word'.
Optional flags a token can have.
Class)
-of an Attribute as an argument and returns an instance. If an Attribute of the same type was previously added, then
-the already existing instance is returned, otherwise a new instance is created and returned. Therefore TokenStreams/-Filters
-can safely call addAttribute() with the same Attribute type multiple times. Even consumers of TokenStreams should
-normally call addAttribute() instead of getAttribute(), because it would not fail if the TokenStream does not have this
-Attribute (getAttribute() would throw an IllegalArgumentException, if the Attribute is missing). More advanced code
-could simply check with hasAttribute(), if a TokenStream has it, and may conditionally leave out processing for
-extra performance.
-
-public class MyAnalyzer extends Analyzer {
-
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream stream = new WhitespaceTokenizer(reader);
- return stream;
- }
-
- public static void main(String[] args) throws IOException {
- // text to tokenize
- final String text = "This is a demo of the new TokenStream API";
-
- MyAnalyzer analyzer = new MyAnalyzer();
- TokenStream stream = analyzer.tokenStream("field", new StringReader(text));
-
- // get the CharTermAttribute from the TokenStream
- CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
-
- stream.reset();
-
- // print all tokens until stream is exhausted
- while (stream.incrementToken()) {
- System.out.println(termAtt.toString());
- }
-
- stream.end()
- stream.close();
- }
-}
-
-In this easy example a simple white space tokenization is performed. In main() a loop consumes the stream and
-prints the term text of the tokens by accessing the CharTermAttribute that the WhitespaceTokenizer provides.
-Here is the output:
--This -is -a -demo -of -the -new -TokenStream -API --
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream stream = new WhitespaceTokenizer(reader);
- stream = new LengthFilter(stream, 3, Integer.MAX_VALUE);
- return stream;
- }
-
-Note how now only words with 3 or more characters are contained in the output:
--This -demo -the -new -TokenStream -API --Now let's take a look how the LengthFilter is implemented (it is part of Lucene's core): -
-public final class LengthFilter extends TokenFilter {
-
- final int min;
- final int max;
-
- private CharTermAttribute termAtt;
-
- /**
- * Build a filter that removes words that are too long or too
- * short from the text.
- */
- public LengthFilter(TokenStream in, int min, int max)
- {
- super(in);
- this.min = min;
- this.max = max;
- termAtt = addAttribute(CharTermAttribute.class);
- }
-
- /**
- * Returns the next input Token whose term() is the right len
- */
- public final boolean incrementToken() throws IOException
- {
- assert termAtt != null;
- // return the first non-stop word found
- while (input.incrementToken()) {
- int len = termAtt.length();
- if (len >= min && len <= max) {
- return true;
- }
- // note: else we ignore it but should we index each part of it?
- }
- // reached EOS -- return null
- return false;
- }
-}
-
-The CharTermAttribute is added in the constructor and stored in the instance variable termAtt.
-Remember that there can only be a single instance of CharTermAttribute in the chain, so in our example the
-addAttribute() call in LengthFilter returns the TermAttribute that the WhitespaceTokenizer already added. The tokens
-are retrieved from the input stream in the incrementToken() method. By looking at the term text
-in the CharTermAttribute the length of the term can be determined and too short or too long tokens are skipped.
-Note how incrementToken() can efficiently access the instance variable; no attribute lookup
-is neccessary. The same is true for the consumer, which can simply use local references to the Attributes.
-
-PartOfSpeechAttribute. First we need to define the interface of the new Attribute:
-
- public interface PartOfSpeechAttribute extends Attribute {
- public static enum PartOfSpeech {
- Noun, Verb, Adjective, Adverb, Pronoun, Preposition, Conjunction, Article, Unknown
- }
-
- public void setPartOfSpeech(PartOfSpeech pos);
-
- public PartOfSpeech getPartOfSpeech();
- }
-
-
-Now we also need to write the implementing class. The name of that class is important here: By default, Lucene
-checks if there is a class with the name of the Attribute with the postfix 'Impl'. In this example, we would
-consequently call the implementing class PartOfSpeechAttributeImpl.
-public final class PartOfSpeechAttributeImpl extends AttributeImpl
- implements PartOfSpeechAttribute{
-
- private PartOfSpeech pos = PartOfSpeech.Unknown;
-
- public void setPartOfSpeech(PartOfSpeech pos) {
- this.pos = pos;
- }
-
- public PartOfSpeech getPartOfSpeech() {
- return pos;
- }
-
- public void clear() {
- pos = PartOfSpeech.Unknown;
- }
-
- public void copyTo(AttributeImpl target) {
- ((PartOfSpeechAttributeImpl) target).pos = pos;
- }
-
- public boolean equals(Object other) {
- if (other == this) {
- return true;
- }
-
- if (other instanceof PartOfSpeechAttributeImpl) {
- return pos == ((PartOfSpeechAttributeImpl) other).pos;
- }
-
- return false;
- }
-
- public int hashCode() {
- return pos.ordinal();
- }
-}
-
-This is a simple Attribute implementation has only a single variable that stores the part-of-speech of a token. It extends the
-new AttributeImpl class and therefore implements its abstract methods clear(), copyTo(), equals(), hashCode().
-Now we need a TokenFilter that can set this new PartOfSpeechAttribute for each token. In this example we show a very naive filter
-that tags every word with a leading upper-case letter as a 'Noun' and all other words as 'Unknown'.
-
- public static class PartOfSpeechTaggingFilter extends TokenFilter {
- PartOfSpeechAttribute posAtt;
- CharTermAttribute termAtt;
-
- protected PartOfSpeechTaggingFilter(TokenStream input) {
- super(input);
- posAtt = addAttribute(PartOfSpeechAttribute.class);
- termAtt = addAttribute(CharTermAttribute.class);
- }
-
- public boolean incrementToken() throws IOException {
- if (!input.incrementToken()) {return false;}
- posAtt.setPartOfSpeech(determinePOS(termAtt.buffer(), 0, termAtt.length()));
- return true;
- }
-
- // determine the part of speech for the given term
- protected PartOfSpeech determinePOS(char[] term, int offset, int length) {
- // naive implementation that tags every uppercased word as noun
- if (length > 0 && Character.isUpperCase(term[0])) {
- return PartOfSpeech.Noun;
- }
- return PartOfSpeech.Unknown;
- }
- }
-
-Just like the LengthFilter, this new filter accesses the attributes it needs in the constructor and
-stores references in instance variables. Notice how you only need to pass in the interface of the new
-Attribute and instantiating the correct class is automatically been taken care of.
-Now we need to add the filter to the chain:
-
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream stream = new WhitespaceTokenizer(reader);
- stream = new LengthFilter(stream, 3, Integer.MAX_VALUE);
- stream = new PartOfSpeechTaggingFilter(stream);
- return stream;
- }
-
-Now let's look at the output:
--This -demo -the -new -TokenStream -API --Apparently it hasn't changed, which shows that adding a custom attribute to a TokenStream/Filter chain does not -affect any existing consumers, simply because they don't know the new Attribute. Now let's change the consumer -to make use of the new PartOfSpeechAttribute and print it out: -
- public static void main(String[] args) throws IOException {
- // text to tokenize
- final String text = "This is a demo of the new TokenStream API";
-
- MyAnalyzer analyzer = new MyAnalyzer();
- TokenStream stream = analyzer.tokenStream("field", new StringReader(text));
-
- // get the CharTermAttribute from the TokenStream
- CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
-
- // get the PartOfSpeechAttribute from the TokenStream
- PartOfSpeechAttribute posAtt = stream.addAttribute(PartOfSpeechAttribute.class);
-
- stream.reset();
-
- // print all tokens until stream is exhausted
- while (stream.incrementToken()) {
- System.out.println(termAtt.toString() + ": " + posAtt.getPartOfSpeech());
- }
-
- stream.end();
- stream.close();
- }
-
-The change that was made is to get the PartOfSpeechAttribute from the TokenStream and print out its contents in
-the while loop that consumes the stream. Here is the new output:
--This: Noun -demo: Unknown -the: Unknown -new: Unknown -TokenStream: Noun -API: Noun --Each word is now followed by its assigned PartOfSpeech tag. Of course this is a naive -part-of-speech tagging. The word 'This' should not even be tagged as noun; it is only spelled capitalized because it -is the first word of a sentence. Actually this is a good opportunity for an excerise. To practice the usage of the new -API the reader could now write an Attribute and TokenFilter that can specify for each word if it was the first token -of a sentence or not. Then the PartOfSpeechTaggingFilter can make use of this knowledge and only tag capitalized words -as nouns if not the first word of a sentence (we know, this is still not a correct behavior, but hey, it's a good exercise). -As a small hint, this is how the new Attribute class could begin: -
- public class FirstTokenOfSentenceAttributeImpl extends Attribute
- implements FirstTokenOfSentenceAttribute {
-
- private boolean firstToken;
-
- public void setFirstToken(boolean firstToken) {
- this.firstToken = firstToken;
- }
-
- public boolean getFirstToken() {
- return firstToken;
- }
-
- public void clear() {
- firstToken = false;
- }
-
- ...
-
-
-
Index: lucene/src/java/org/apache/lucene/LucenePackage.java
===================================================================
--- lucene/src/java/org/apache/lucene/LucenePackage.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/LucenePackage.java (working copy)
@@ -1,29 +0,0 @@
-package org.apache.lucene;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Lucene's package information, including version. **/
-public final class LucenePackage {
-
- private LucenePackage() {} // can't construct
-
- /** Return Lucene's package, including version information. */
- public static Package get() {
- return LucenePackage.class.getPackage();
- }
-}
Index: lucene/src/java/org/apache/lucene/index/DocInverterPerField.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (working copy)
@@ -185,7 +185,7 @@
}
}
- fieldState.offset += docState.analyzer.getOffsetGap(field);
+ fieldState.offset += docState.analyzer.getOffsetGap(field.name(), field.isTokenized());
fieldState.boost *= field.getBoost();
}
Index: lucene/src/java/org/apache/lucene/index/Payload.java
===================================================================
--- lucene/src/java/org/apache/lucene/index/Payload.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/index/Payload.java (working copy)
@@ -1,199 +0,0 @@
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.util.ArrayUtil;
-
-/**
- * A Payload is metadata that can be stored together with each occurrence
- * of a term. This metadata is stored inline in the posting list of the
- * specific term.
- * - * To store payloads in the index a {@link TokenStream} has to be used that - * produces payload data. - *
- * Use {@link DocsAndPositionsEnum#getPayload()}
- * to retrieve the payloads from the index.
- *
- */
-public class Payload implements Cloneable {
- /** the byte array containing the payload data */
- protected byte[] data;
-
- /** the offset within the byte array */
- protected int offset;
-
- /** the length of the payload data */
- protected int length;
-
- /** Creates an empty payload and does not allocate a byte array. */
- public Payload() {
- // nothing to do
- }
-
- /**
- * Creates a new payload with the the given array as data.
- * A reference to the passed-in array is held, i. e. no
- * copy is made.
- *
- * @param data the data of this payload
- */
- public Payload(byte[] data) {
- this(data, 0, data.length);
- }
-
- /**
- * Creates a new payload with the the given array as data.
- * A reference to the passed-in array is held, i. e. no
- * copy is made.
- *
- * @param data the data of this payload
- * @param offset the offset in the data byte array
- * @param length the length of the data
- */
- public Payload(byte[] data, int offset, int length) {
- if (offset < 0 || offset + length > data.length) {
- throw new IllegalArgumentException();
- }
- this.data = data;
- this.offset = offset;
- this.length = length;
- }
-
- /**
- * Sets this payloads data.
- * A reference to the passed-in array is held, i. e. no
- * copy is made.
- */
- public void setData(byte[] data) {
- setData(data, 0, data.length);
- }
-
- /**
- * Sets this payloads data.
- * A reference to the passed-in array is held, i. e. no
- * copy is made.
- */
- public void setData(byte[] data, int offset, int length) {
- this.data = data;
- this.offset = offset;
- this.length = length;
- }
-
- /**
- * Returns a reference to the underlying byte array
- * that holds this payloads data.
- */
- public byte[] getData() {
- return this.data;
- }
-
- /**
- * Returns the offset in the underlying byte array
- */
- public int getOffset() {
- return this.offset;
- }
-
- /**
- * Returns the length of the payload data.
- */
- public int length() {
- return this.length;
- }
-
- /**
- * Returns the byte at the given index.
- */
- public byte byteAt(int index) {
- if (0 <= index && index < this.length) {
- return this.data[this.offset + index];
- }
- throw new ArrayIndexOutOfBoundsException(index);
- }
-
- /**
- * Allocates a new byte array, copies the payload data into it and returns it.
- */
- public byte[] toByteArray() {
- byte[] retArray = new byte[this.length];
- System.arraycopy(this.data, this.offset, retArray, 0, this.length);
- return retArray;
- }
-
- /**
- * Copies the payload data to a byte array.
- *
- * @param target the target byte array
- * @param targetOffset the offset in the target byte array
- */
- public void copyTo(byte[] target, int targetOffset) {
- if (this.length > target.length + targetOffset) {
- throw new ArrayIndexOutOfBoundsException();
- }
- System.arraycopy(this.data, this.offset, target, targetOffset, this.length);
- }
-
- /**
- * Clones this payload by creating a copy of the underlying
- * byte array.
- */
- @Override
- public Object clone() {
- try {
- // Start with a shallow copy of data
- Payload clone = (Payload) super.clone();
- // Only copy the part of data that belongs to this Payload
- if (offset == 0 && length == data.length) {
- // It is the whole thing, so just clone it.
- clone.data = data.clone();
- }
- else {
- // Just get the part
- clone.data = this.toByteArray();
- clone.offset = 0;
- }
- return clone;
- } catch (CloneNotSupportedException e) {
- throw new RuntimeException(e); // shouldn't happen
- }
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (obj instanceof Payload) {
- Payload other = (Payload) obj;
- if (length == other.length) {
- for(int i=0;i
IndexWriter creates and maintains an index.
Index: lucene/src/java/org/apache/lucene/store/MMapDirectory.java
===================================================================
--- lucene/src/java/org/apache/lucene/store/MMapDirectory.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/store/MMapDirectory.java (working copy)
@@ -31,6 +31,7 @@
import java.security.PrivilegedActionException;
import java.lang.reflect.Method;
+import org.apache.lucene.util.AlreadyClosedException;
import org.apache.lucene.util.Constants;
/** File-based {@link Directory} implementation that uses
Index: lucene/src/java/org/apache/lucene/store/Directory.java
===================================================================
--- lucene/src/java/org/apache/lucene/store/Directory.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/store/Directory.java (working copy)
@@ -22,6 +22,7 @@
import java.io.Closeable;
import java.util.Collection; // for javadocs
+import org.apache.lucene.util.AlreadyClosedException;
import org.apache.lucene.util.IOUtils;
/** A Directory is a flat list of files. Files may be written once, when they
Index: lucene/src/java/org/apache/lucene/store/AlreadyClosedException.java
===================================================================
--- lucene/src/java/org/apache/lucene/store/AlreadyClosedException.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/store/AlreadyClosedException.java (working copy)
@@ -1,28 +0,0 @@
-package org.apache.lucene.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This exception is thrown when there is an attempt to
- * access something that has already been closed.
- */
-public class AlreadyClosedException extends IllegalStateException {
- public AlreadyClosedException(String message) {
- super(message);
- }
-}
Index: lucene/src/java/org/apache/lucene/util/Attribute.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/Attribute.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/util/Attribute.java (working copy)
@@ -1,24 +0,0 @@
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Base interface for attributes.
- */
-public interface Attribute {
-}
Index: lucene/src/java/org/apache/lucene/util/AttributeReflector.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/AttributeReflector.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/util/AttributeReflector.java (working copy)
@@ -1,34 +0,0 @@
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This interface is used to reflect contents of {@link AttributeSource} or {@link AttributeImpl}.
- */
-public interface AttributeReflector {
-
- /**
- * This method gets called for every property in an {@link AttributeImpl}/{@link AttributeSource}
- * passing the class name of the {@link Attribute}, a key and the actual value.
- * E.g., an invocation of {@link org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl#reflectWith}
- * would call this method once using {@code org.apache.lucene.analysis.tokenattributes.CharTermAttribute.class}
- * as attribute class, {@code "term"} as key and the actual value as a String.
- */
- public void reflect(Class attClass, String key, Object value);
-
-}
Index: lucene/src/java/org/apache/lucene/util/SorterTemplate.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/SorterTemplate.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/util/SorterTemplate.java (working copy)
@@ -1,212 +0,0 @@
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This class was inspired by CGLIB, but provides a better
- * QuickSort algorithm without additional InsertionSort
- * at the end.
- * To use, subclass and override the four abstract methods
- * which compare and modify your data.
- * Allows custom swap so that two arrays can be sorted
- * at the same time.
- * @lucene.internal
- */
-public abstract class SorterTemplate {
-
- private static final int MERGESORT_THRESHOLD = 12;
- private static final int QUICKSORT_THRESHOLD = 7;
-
- /** Implement this method, that swaps slots {@code i} and {@code j} in your data */
- protected abstract void swap(int i, int j);
-
- /** Compares slots {@code i} and {@code j} of you data.
- * Should be implemented like valueOf(i).compareTo(valueOf(j)) */
- protected abstract int compare(int i, int j);
-
- /** Implement this method, that stores the value of slot {@code i} as pivot value */
- protected abstract void setPivot(int i);
-
- /** Implements the compare function for the previously stored pivot value.
- * Should be implemented like pivot.compareTo(valueOf(j)) */
- protected abstract int comparePivot(int j);
-
- /** Sorts via stable in-place InsertionSort algorithm
- *(ideal for small collections which are mostly presorted). */
- public final void insertionSort(int lo, int hi) {
- for (int i = lo + 1 ; i <= hi; i++) {
- for (int j = i; j > lo; j--) {
- if (compare(j - 1, j) > 0) {
- swap(j - 1, j);
- } else {
- break;
- }
- }
- }
- }
-
- /** Sorts via in-place, but unstable, QuickSort algorithm.
- * For small collections falls back to {@link #insertionSort(int,int)}. */
- public final void quickSort(final int lo, final int hi) {
- if (hi <= lo) return;
- // from Integer's Javadocs: ceil(log2(x)) = 32 - numberOfLeadingZeros(x - 1)
- quickSort(lo, hi, (Integer.SIZE - Integer.numberOfLeadingZeros(hi - lo)) << 1);
- }
-
- private void quickSort(int lo, int hi, int maxDepth) {
- // fall back to insertion when array has short length
- final int diff = hi - lo;
- if (diff <= QUICKSORT_THRESHOLD) {
- insertionSort(lo, hi);
- return;
- }
-
- // fall back to merge sort when recursion depth gets too big
- if (--maxDepth == 0) {
- mergeSort(lo, hi);
- return;
- }
-
- final int mid = lo + (diff >>> 1);
-
- if (compare(lo, mid) > 0) {
- swap(lo, mid);
- }
-
- if (compare(mid, hi) > 0) {
- swap(mid, hi);
- if (compare(lo, mid) > 0) {
- swap(lo, mid);
- }
- }
-
- int left = lo + 1;
- int right = hi - 1;
-
- setPivot(mid);
- for (;;) {
- while (comparePivot(right) < 0)
- --right;
-
- while (left < right && comparePivot(left) >= 0)
- ++left;
-
- if (left < right) {
- swap(left, right);
- --right;
- } else {
- break;
- }
- }
-
- quickSort(lo, left, maxDepth);
- quickSort(left + 1, hi, maxDepth);
- }
-
- /** Sorts via stable in-place MergeSort algorithm
- * For small collections falls back to {@link #insertionSort(int,int)}. */
- public final void mergeSort(int lo, int hi) {
- final int diff = hi - lo;
- if (diff <= MERGESORT_THRESHOLD) {
- insertionSort(lo, hi);
- return;
- }
-
- final int mid = lo + (diff >>> 1);
-
- mergeSort(lo, mid);
- mergeSort(mid, hi);
- merge(lo, mid, hi, mid - lo, hi - mid);
- }
-
- private void merge(int lo, int pivot, int hi, int len1, int len2) {
- if (len1 == 0 || len2 == 0) {
- return;
- }
- if (len1 + len2 == 2) {
- if (compare(pivot, lo) < 0) {
- swap(pivot, lo);
- }
- return;
- }
- int first_cut, second_cut;
- int len11, len22;
- if (len1 > len2) {
- len11 = len1 >>> 1;
- first_cut = lo + len11;
- second_cut = lower(pivot, hi, first_cut);
- len22 = second_cut - pivot;
- } else {
- len22 = len2 >>> 1;
- second_cut = pivot + len22;
- first_cut = upper(lo, pivot, second_cut);
- len11 = first_cut - lo;
- }
- rotate(first_cut, pivot, second_cut);
- final int new_mid = first_cut + len22;
- merge(lo, first_cut, new_mid, len11, len22);
- merge(new_mid, second_cut, hi, len1 - len11, len2 - len22);
- }
-
- private void rotate(int lo, int mid, int hi) {
- int lot = lo;
- int hit = mid - 1;
- while (lot < hit) {
- swap(lot++, hit--);
- }
- lot = mid; hit = hi - 1;
- while (lot < hit) {
- swap(lot++, hit--);
- }
- lot = lo; hit = hi - 1;
- while (lot < hit) {
- swap(lot++, hit--);
- }
- }
-
- private int lower(int lo, int hi, int val) {
- int len = hi - lo;
- while (len > 0) {
- final int half = len >>> 1,
- mid = lo + half;
- if (compare(mid, val) < 0) {
- lo = mid + 1;
- len = len - half -1;
- } else {
- len = half;
- }
- }
- return lo;
- }
-
- private int upper(int lo, int hi, int val) {
- int len = hi - lo;
- while (len > 0) {
- final int half = len >>> 1,
- mid = lo + half;
- if (compare(val, mid) < 0) {
- len = half;
- } else {
- lo = mid + 1;
- len = len - half -1;
- }
- }
- return lo;
- }
-
-}
Index: lucene/src/java/org/apache/lucene/util/Constants.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/Constants.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/util/Constants.java (working copy)
@@ -1,89 +0,0 @@
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.LucenePackage;
-
-/**
- * Some useful constants.
- **/
-
-public final class Constants {
- private Constants() {} // can't construct
-
- /** The value of System.getProperty("java.version"). **/
- public static final String JAVA_VERSION = System.getProperty("java.version");
- /** True iff this is Java version 1.1. */
- public static final boolean JAVA_1_1 = JAVA_VERSION.startsWith("1.1.");
- /** True iff this is Java version 1.2. */
- public static final boolean JAVA_1_2 = JAVA_VERSION.startsWith("1.2.");
- /** True iff this is Java version 1.3. */
- public static final boolean JAVA_1_3 = JAVA_VERSION.startsWith("1.3.");
-
- /** The value of System.getProperty("os.name"). **/
- public static final String OS_NAME = System.getProperty("os.name");
- /** True iff running on Linux. */
- public static final boolean LINUX = OS_NAME.startsWith("Linux");
- /** True iff running on Windows. */
- public static final boolean WINDOWS = OS_NAME.startsWith("Windows");
- /** True iff running on SunOS. */
- public static final boolean SUN_OS = OS_NAME.startsWith("SunOS");
-
- public static final String OS_ARCH = System.getProperty("os.arch");
- public static final String OS_VERSION = System.getProperty("os.version");
- public static final String JAVA_VENDOR = System.getProperty("java.vendor");
-
- // NOTE: this logic may not be correct; if you know of a
- // more reliable approach please raise it on java-dev!
- public static final boolean JRE_IS_64BIT;
- static {
- String x = System.getProperty("sun.arch.data.model");
- if (x != null) {
- JRE_IS_64BIT = x.indexOf("64") != -1;
- } else {
- if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) {
- JRE_IS_64BIT = true;
- } else {
- JRE_IS_64BIT = false;
- }
- }
- }
-
- // this method prevents inlining the final version constant in compiled classes,
- // see: http://www.javaworld.com/community/node/3400
- private static String ident(final String s) {
- return s.toString();
- }
-
- // NOTE: we track per-segment version as a String with the "X.Y" format, e.g.
- // "4.0", "3.1", "3.0". Therefore when we change this constant, we should keep
- // the format.
- public static final String LUCENE_MAIN_VERSION = ident("4.0");
-
- public static final String LUCENE_VERSION;
- static {
- Package pkg = LucenePackage.get();
- String v = (pkg == null) ? null : pkg.getImplementationVersion();
- if (v == null) {
- v = LUCENE_MAIN_VERSION + "-SNAPSHOT";
- } else if (!v.startsWith(LUCENE_MAIN_VERSION)) {
- v = LUCENE_MAIN_VERSION + "-SNAPSHOT " + v;
- }
- LUCENE_VERSION = ident(v);
- }
-}
Index: lucene/src/java/org/apache/lucene/util/MemoryModel.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/MemoryModel.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/util/MemoryModel.java (working copy)
@@ -1,48 +0,0 @@
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-/**
- * Returns primitive memory sizes for estimating RAM usage.
- *
- */
-public abstract class MemoryModel {
-
- /**
- * @return size of array beyond contents
- */
- public abstract int getArraySize();
-
- /**
- * @return Class size overhead
- */
- public abstract int getClassSize();
-
- /**
- * @param clazz a primitive Class - bool, byte, char, short, long, float,
- * short, double, int
- * @return the size in bytes of given primitive Class
- */
- public abstract int getPrimitiveSize(Class clazz);
-
- /**
- * @return size of reference
- */
- public abstract int getReferenceSize();
-
-}
Index: lucene/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java (working copy)
@@ -1,78 +0,0 @@
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.util.IdentityHashMap;
-import java.util.Map;
-
-/**
- * An average, best guess, MemoryModel that should work okay on most systems.
- *
- */
-public class AverageGuessMemoryModel extends MemoryModel {
- // best guess primitive sizes
- private final Map
- * int hash = 0;
- * for (int i = offset; i < offset + length; i++) {
- * hash = 31*hash + bytes[i];
- * }
- *
- */
- @Override
- public int hashCode() {
- int result = 0;
- final int end = offset + length;
- for(int i=offset;iTo quickly execute range queries in Apache Lucene, a range is divided recursively - * into multiple intervals for searching: The center of the range is searched only with - * the lowest possible precision in the trie, while the boundaries are matched - * more exactly. This reduces the number of terms dramatically. - * - *
This class generates terms to achieve this: First the numerical integer values need to
- * be converted to bytes. For that integer values (32 bit or 64 bit) are made unsigned
- * and the bits are converted to ASCII chars with each 7 bit. The resulting byte[] is
- * sortable like the original integer value (even using UTF-8 sort order). Each value is also
- * prefixed (in the first char) by the shift value (number of bits removed) used
- * during encoding.
- *
- *
To also index floating point numbers, this class supplies two methods to convert them - * to integer values by changing their bit layout: {@link #doubleToSortableLong}, - * {@link #floatToSortableInt}. You will have no precision loss by - * converting floating point numbers to integers and back (only that the integer form - * is not usable). Other data types like dates can easily converted to longs or ints (e.g. - * date to long: {@link java.util.Date#getTime}). - * - *
For easy usage, the trie algorithm is implemented for indexing inside
- * {@link NumericTokenStream} that can index int, long,
- * float, and double. For querying,
- * {@link NumericRangeQuery} and {@link NumericRangeFilter} implement the query part
- * for the same data types.
- *
- *
This class can also be used, to generate lexicographically sortable (according to
- * {@link BytesRef#getUTF8SortedAsUTF16Comparator()}) representations of numeric data
- * types for other usages (e.g. sorting).
- *
- * @lucene.internal
- * @since 2.9, API changed non backwards-compliant in 4.0
- */
-public final class NumericUtils {
-
- private NumericUtils() {} // no instance!
-
- /**
- * The default precision step used by {@link NumericField}, {@link NumericTokenStream},
- * {@link NumericRangeQuery}, and {@link NumericRangeFilter} as default
- */
- public static final int PRECISION_STEP_DEFAULT = 4;
-
- /**
- * Longs are stored at lower precision by shifting off lower bits. The shift count is
- * stored as This method is used by {@link NumericRangeQuery}.
- */
- public static void splitLongRange(final LongRangeBuilder builder,
- final int precisionStep, final long minBound, final long maxBound
- ) {
- splitRange(builder, 64, precisionStep, minBound, maxBound);
- }
-
- /**
- * Splits an int range recursively.
- * You may implement a builder that adds clauses to a
- * {@link org.apache.lucene.search.BooleanQuery} for each call to its
- * {@link IntRangeBuilder#addRange(BytesRef,BytesRef)}
- * method.
- * This method is used by {@link NumericRangeQuery}.
- */
- public static void splitIntRange(final IntRangeBuilder builder,
- final int precisionStep, final int minBound, final int maxBound
- ) {
- splitRange(builder, 32, precisionStep, minBound, maxBound);
- }
-
- /** This helper does the splitting for both 32 and 64 bit. */
- private static void splitRange(
- final Object builder, final int valSize,
- final int precisionStep, long minBound, long maxBound
- ) {
- if (precisionStep < 1)
- throw new IllegalArgumentException("precisionStep must be >=1");
- if (minBound > maxBound) return;
- for (int shift=0; ; shift += precisionStep) {
- // calculate new bounds for inner precision
- final long diff = 1L << (shift+precisionStep),
- mask = ((1L< Please note: It is not guaranteed, that
- * Note that this method does not affect attributes of the targetStream
- * that are not contained in this state. In other words, if for example
- * the targetStream contains an OffsetAttribute, but this state doesn't, then
- * the value of the OffsetAttribute remains unchanged. It might be desirable to
- * reset its value to the default, in which case the caller should first
- * call {@link TokenStream#clearAttributes()} on the targetStream.
- */
- public final void restoreState(State state) {
- if (state == null) return;
-
- do {
- AttributeImpl targetImpl = attributeImpls.get(state.attribute.getClass());
- if (targetImpl == null) {
- throw new IllegalArgumentException("State contains AttributeImpl of type " +
- state.attribute.getClass().getName() + " that is not in in this AttributeSource");
- }
- state.attribute.copyTo(targetImpl);
- state = state.next;
- } while (state != null);
- }
-
- @Override
- public int hashCode() {
- int code = 0;
- for (State state = getCurrentState(); state != null; state = state.next) {
- code = code * 31 + state.attribute.hashCode();
- }
- return code;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
-
- if (obj instanceof AttributeSource) {
- AttributeSource other = (AttributeSource) obj;
-
- if (hasAttributes()) {
- if (!other.hasAttributes()) {
- return false;
- }
-
- if (this.attributeImpls.size() != other.attributeImpls.size()) {
- return false;
- }
-
- // it is only equal if all attribute impls are the same in the same order
- State thisState = this.getCurrentState();
- State otherState = other.getCurrentState();
- while (thisState != null && otherState != null) {
- if (otherState.attribute.getClass() != thisState.attribute.getClass() || !otherState.attribute.equals(thisState.attribute)) {
- return false;
- }
- thisState = thisState.next;
- otherState = otherState.next;
- }
- return true;
- } else {
- return !other.hasAttributes();
- }
- } else
- return false;
- }
-
- /**
- * This method returns the current attribute values as a string in the following format
- * by calling the {@link #reflectWith(AttributeReflector)} method:
- *
- * This method iterates over all Attribute implementations and calls the
- * corresponding {@link AttributeImpl#reflectWith} method.
- * WARNING: This is not a valid UTF8 Term
- **/
- public static final BytesRef BIG_TERM = new BytesRef(
- new byte[] {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1}
- ); // TODO this is unrelated here find a better place for it
-
- public static void main(String[] args) {
- System.out.println(Character.toChars(0x10FFFF + 1));
- }
-
- private UnicodeUtil() {} // no instance
-
- public static final int UNI_SUR_HIGH_START = 0xD800;
- public static final int UNI_SUR_HIGH_END = 0xDBFF;
- public static final int UNI_SUR_LOW_START = 0xDC00;
- public static final int UNI_SUR_LOW_END = 0xDFFF;
- public static final int UNI_REPLACEMENT_CHAR = 0xFFFD;
-
- private static final long UNI_MAX_BMP = 0x0000FFFF;
-
- private static final long HALF_SHIFT = 10;
- private static final long HALF_MASK = 0x3FFL;
-
- private static final int SURROGATE_OFFSET =
- Character.MIN_SUPPLEMENTARY_CODE_POINT -
- (UNI_SUR_HIGH_START << HALF_SHIFT) - UNI_SUR_LOW_START;
-
- /** Encode characters from a char[] source, starting at
- * offset for length chars. Returns a hash of the resulting bytes. After encoding, result.offset will always be 0. */
- public static int UTF16toUTF8WithHash(final char[] source, final int offset, final int length, BytesRef result) {
- int hash = 0;
- int upto = 0;
- int i = offset;
- final int end = offset + length;
- byte[] out = result.bytes;
- // Pre-allocate for worst case 4-for-1
- final int maxLen = length * 4;
- if (out.length < maxLen)
- out = result.bytes = new byte[ArrayUtil.oversize(maxLen, 1)];
- result.offset = 0;
-
- while(i < end) {
-
- final int code = (int) source[i++];
-
- if (code < 0x80) {
- hash = 31*hash + (out[upto++] = (byte) code);
- } else if (code < 0x800) {
- hash = 31*hash + (out[upto++] = (byte) (0xC0 | (code >> 6)));
- hash = 31*hash + (out[upto++] = (byte)(0x80 | (code & 0x3F)));
- } else if (code < 0xD800 || code > 0xDFFF) {
- hash = 31*hash + (out[upto++] = (byte)(0xE0 | (code >> 12)));
- hash = 31*hash + (out[upto++] = (byte)(0x80 | ((code >> 6) & 0x3F)));
- hash = 31*hash + (out[upto++] = (byte)(0x80 | (code & 0x3F)));
- } else {
- // surrogate pair
- // confirm valid high surrogate
- if (code < 0xDC00 && i < end) {
- int utf32 = (int) source[i];
- // confirm valid low surrogate and write pair
- if (utf32 >= 0xDC00 && utf32 <= 0xDFFF) {
- utf32 = (code << 10) + utf32 + SURROGATE_OFFSET;
- i++;
- hash = 31*hash + (out[upto++] = (byte)(0xF0 | (utf32 >> 18)));
- hash = 31*hash + (out[upto++] = (byte)(0x80 | ((utf32 >> 12) & 0x3F)));
- hash = 31*hash + (out[upto++] = (byte)(0x80 | ((utf32 >> 6) & 0x3F)));
- hash = 31*hash + (out[upto++] = (byte)(0x80 | (utf32 & 0x3F)));
- continue;
- }
- }
- // replace unpaired surrogate or out-of-order low surrogate
- // with substitution character
- hash = 31*hash + (out[upto++] = (byte) 0xEF);
- hash = 31*hash + (out[upto++] = (byte) 0xBF);
- hash = 31*hash + (out[upto++] = (byte) 0xBD);
- }
- }
- //assert matches(source, offset, length, out, upto);
- result.length = upto;
- return hash;
- }
-
- /** Encode characters from a char[] source, starting at
- * offset for length chars. After encoding, result.offset will always be 0.
- */
- public static void UTF16toUTF8(final char[] source, final int offset, final int length, BytesRef result) {
-
- int upto = 0;
- int i = offset;
- final int end = offset + length;
- byte[] out = result.bytes;
- // Pre-allocate for worst case 4-for-1
- final int maxLen = length * 4;
- if (out.length < maxLen)
- out = result.bytes = new byte[maxLen];
- result.offset = 0;
-
- while(i < end) {
-
- final int code = (int) source[i++];
-
- if (code < 0x80)
- out[upto++] = (byte) code;
- else if (code < 0x800) {
- out[upto++] = (byte) (0xC0 | (code >> 6));
- out[upto++] = (byte)(0x80 | (code & 0x3F));
- } else if (code < 0xD800 || code > 0xDFFF) {
- out[upto++] = (byte)(0xE0 | (code >> 12));
- out[upto++] = (byte)(0x80 | ((code >> 6) & 0x3F));
- out[upto++] = (byte)(0x80 | (code & 0x3F));
- } else {
- // surrogate pair
- // confirm valid high surrogate
- if (code < 0xDC00 && i < end) {
- int utf32 = (int) source[i];
- // confirm valid low surrogate and write pair
- if (utf32 >= 0xDC00 && utf32 <= 0xDFFF) {
- utf32 = (code << 10) + utf32 + SURROGATE_OFFSET;
- i++;
- out[upto++] = (byte)(0xF0 | (utf32 >> 18));
- out[upto++] = (byte)(0x80 | ((utf32 >> 12) & 0x3F));
- out[upto++] = (byte)(0x80 | ((utf32 >> 6) & 0x3F));
- out[upto++] = (byte)(0x80 | (utf32 & 0x3F));
- continue;
- }
- }
- // replace unpaired surrogate or out-of-order low surrogate
- // with substitution character
- out[upto++] = (byte) 0xEF;
- out[upto++] = (byte) 0xBF;
- out[upto++] = (byte) 0xBD;
- }
- }
- //assert matches(source, offset, length, out, upto);
- result.length = upto;
- }
-
- /** Encode characters from this String, starting at offset
- * for length characters. After encoding, result.offset will always be 0.
- */
- public static void UTF16toUTF8(final CharSequence s, final int offset, final int length, BytesRef result) {
- final int end = offset + length;
-
- byte[] out = result.bytes;
- result.offset = 0;
- // Pre-allocate for worst case 4-for-1
- final int maxLen = length * 4;
- if (out.length < maxLen)
- out = result.bytes = new byte[maxLen];
-
- int upto = 0;
- for(int i=offset;i
- * NOTE: Full characters are read, even if this reads past the length passed (and
- * can result in an ArrayOutOfBoundsException if invalid UTF-8 is passed).
- * Explicit checks for valid UTF-8 are not performed.
- */
- public static void UTF8toUTF16(byte[] utf8, int offset, int length, CharsRef chars) {
- int out_offset = chars.offset = 0;
- final char[] out = chars.chars = ArrayUtil.grow(chars.chars, length);
- final int limit = offset + length;
- while (offset < limit) {
- int b = utf8[offset++]&0xff;
- if (b < 0xc0) {
- assert b < 0x80;
- out[out_offset++] = (char)b;
- } else if (b < 0xe0) {
- out[out_offset++] = (char)(((b&0x1f)<<6) + (utf8[offset++]&0x3f));
- } else if (b < 0xf0) {
- out[out_offset++] = (char)(((b&0xf)<<12) + ((utf8[offset]&0x3f)<<6) + (utf8[offset+1]&0x3f));
- offset += 2;
- } else {
- assert b < 0xf8;
- int ch = ((b&0x7)<<18) + ((utf8[offset]&0x3f)<<12) + ((utf8[offset+1]&0x3f)<<6) + (utf8[offset+2]&0x3f);
- offset += 3;
- if (ch < UNI_MAX_BMP) {
- out[out_offset++] = (char)ch;
- } else {
- int chHalf = ch - 0x0010000;
- out[out_offset++] = (char) ((chHalf >> 10) + 0xD800);
- out[out_offset++] = (char) ((chHalf & HALF_MASK) + 0xDC00);
- }
- }
- }
- chars.length = out_offset - chars.offset;
- }
-
- /**
- * Utility method for {@link #UTF8toUTF16(byte[], int, int, CharsRef)}
- * @see #UTF8toUTF16(byte[], int, int, CharsRef)
- */
- public static void UTF8toUTF16(BytesRef bytesRef, CharsRef chars) {
- UTF8toUTF16(bytesRef.bytes, bytesRef.offset, bytesRef.length, chars);
- }
-
-}
Index: lucene/src/java/org/apache/lucene/util/AttributeImpl.java
===================================================================
--- lucene/src/java/org/apache/lucene/util/AttributeImpl.java (revision 1128767)
+++ lucene/src/java/org/apache/lucene/util/AttributeImpl.java (working copy)
@@ -1,135 +0,0 @@
-package org.apache.lucene.util;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.lang.ref.WeakReference;
-import java.util.LinkedList;
-
-/**
- * Base class for Attributes that can be added to a
- * {@link org.apache.lucene.util.AttributeSource}.
- *
- * Attributes are used to add data in a dynamic, yet type-safe way to a source
- * of usually streamed objects, e. g. a {@link org.apache.lucene.analysis.TokenStream}.
- */
-public abstract class AttributeImpl implements Cloneable, Attribute {
- /**
- * Clears the values in this AttributeImpl and resets it to its
- * default value. If this implementation implements more than one Attribute interface
- * it clears all.
- */
- public abstract void clear();
-
- /**
- * This method returns the current attribute values as a string in the following format
- * by calling the {@link #reflectWith(AttributeReflector)} method:
- *
- * The default implementation calls {@link AttributeReflector#reflect} for all
- * non-static fields from the implementing class, using the field name as key
- * and the field value as value. The Attribute class is also determined by reflection.
- * Please note that the default implementation can only handle single-Attribute
- * implementations.
- *
- * Custom implementations look like this (e.g. for a combined attribute implementation):
- * If you implement this method, make sure that for each invocation, the same set of {@link Attribute}
- * interfaces and keys are passed to {@link AttributeReflector#reflect} in the same order, but possibly
- * different values. So don't automatically exclude e.g. {@code null} properties!
- *
- * @see #reflectAsString(boolean)
- */
- public void reflectWith(AttributeReflector reflector) {
- final Class clazz = this.getClass();
- final LinkedList Note that for simple usage, NumericField is
+ * recommended. NumericField disables norms and
+ * term freqs, as they are not usually needed during
+ * searching. If you need to change these settings, you
+ * should use this class.
+ * See NumericField for capabilities of fields
+ * indexed numerically. Here's an example usage, for an For optimal performance, re-use the TokenStream and Field instance
+ * for more than one document:
+ * This stream is not intended to be used in analyzers;
+ * it's more for iterating the different precisions during
+ * indexing a specific numeric value. NOTE: as token streams are only consumed once
+ * the document is added to the index, if you index more
+ * than one numeric field, use a separate See NumericRangeQuery in Lucene core for more details on the
+ *
+ The start and end offsets permit applications to re-associate a token with
+ its source text, e.g., to display highlighted query terms in a document
+ browser, or to show matching text fragments in a KWIC
+ display, etc.
+
+ The type is a string, assigned by a lexical analyzer
+ (a.k.a. tokenizer), naming the lexical or syntactic class that the token
+ belongs to. For example an end of sentence marker token might be implemented
+ with type "eos". The default token type is "word".
+
+ A Token can optionally have metadata (a.k.a. Payload) in the form of a variable
+ length byte array. For instance, Lucene's DocsAndPositionsEnum#getPayload() can be used
+ to retrieve the
+ payloads from the index.
+
+ NOTE: As of 2.9, Token implements all {@link Attribute} interfaces
+ that are part of core Lucene and can be found in the {@code tokenattributes} subpackage.
+ Even though it is not necessary to use Token anymore, with the new TokenStream API it can
+ be used as convenience class that implements all {@link Attribute}s, which is especially useful
+ to easily switch from the old to the new TokenStream API.
+
+ Tokenizers and TokenFilters should try to re-use a Token
+ instance when possible for best performance, by
+ implementing the {@link TokenStream#incrementToken()} API.
+ Failing that, to create a new Token you should first use
+ one of the constructors that starts with null text. To load
+ the token from a char[] use {@link #copyBuffer(char[], int, int)}.
+ To load from a String use {@link #setEmpty} followed by {@link #append(CharSequence)} or {@link #append(CharSequence, int, int)}.
+ Alternatively you can get the Token's termBuffer by calling either {@link #buffer()},
+ if you know that your text is shorter than the capacity of the termBuffer
+ or {@link #resizeBuffer(int)}, if there is any possibility
+ that you may need to grow the buffer. Fill in the characters of your term into this
+ buffer, with {@link String#getChars(int, int, char[], int)} if loading from a string,
+ or with {@link System#arraycopy(Object, int, Object, int, int)}, and finally call {@link #setLength(int)} to
+ set the length of the term text. See LUCENE-969
+ for details. Typical Token reuse patterns:
+
+ Please note: With Lucene 3.1, the The default value is one.
+ *
+ * Some common uses for this are: CachingTokenFilter implements the optional method
+ * {@link TokenStream#reset()}, which repositions the
+ * stream to the first Token.
+ */
+public final class CachingTokenFilter extends TokenFilter {
+ private List The {@code Analyzer}-API in Lucene is based on the decorator pattern.
+ * Therefore all non-abstract subclasses must be final or their {@link #tokenStream}
+ * and {@link #reusableTokenStream} implementations must be final! This is checked
+ * when Java assertions are enabled.
+ */
+public abstract class Analyzer implements Closeable {
+
+ protected Analyzer() {
+ super();
+ assert assertFinal();
+ }
+
+ private boolean assertFinal() {
+ try {
+ final Class clazz = getClass();
+ assert clazz.isAnonymousClass() ||
+ (clazz.getModifiers() & (Modifier.FINAL | Modifier.PRIVATE)) != 0 ||
+ (
+ Modifier.isFinal(clazz.getMethod("tokenStream", String.class, Reader.class).getModifiers()) &&
+ Modifier.isFinal(clazz.getMethod("reusableTokenStream", String.class, Reader.class).getModifiers())
+ ) :
+ "Analyzer implementation classes or at least their tokenStream() and reusableTokenStream() implementations must be final";
+ return true;
+ } catch (NoSuchMethodException nsme) {
+ return false;
+ }
+ }
+
+ /**
+ * Creates a TokenStream which tokenizes all the text in the provided
+ * Reader. Must be able to handle null field name for
+ * backward compatibility.
+ */
+ public abstract TokenStream tokenStream(String fieldName, Reader reader);
+
+ /**
+ * Creates a TokenStream that is allowed to be re-used
+ * from the previous time that the same thread called
+ * this method. Callers that do not need to use more
+ * than one TokenStream at the same time from this
+ * analyzer should use this method for better
+ * performance.
+ */
+ public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
+ return tokenStream(fieldName, reader);
+ }
+
+ private CloseableThreadLocal
+ The synergy between {@link org.apache.lucene.analysis.Analyzer} and {@link org.apache.lucene.analysis.Tokenizer}
+ is sometimes confusing. To ease on this confusion, some clarifications:
+
+ Lucene Java provides a number of analysis capabilities, the most commonly used one being the StandardAnalyzer.
+ Many applications will have a long and industrious life with nothing more
+ than the StandardAnalyzer. However, there are a few other classes/packages that are worth mentioning:
+
+ Analysis is one of the main causes of performance degradation during indexing. Simply put, the more you analyze the slower the indexing (in most cases).
+ Perhaps your application would be just fine using the simple WhitespaceTokenizer combined with a StopFilter. The contrib/benchmark library can be useful
+ for testing out the speed of the analysis process.
+
+ Applications usually do not invoke analysis – Lucene does it for them:
+
+ Selecting the "correct" analyzer is crucial
+ for search quality, and can also affect indexing and search performance.
+ The "correct" analyzer differs between applications.
+ Lucene java's wiki page
+ AnalysisParalysis
+ provides some data on "analyzing your analyzer".
+ Here are some rules of thumb:
+ SHIFT_START_LONG+shift in the first byte
- */
- public static final byte SHIFT_START_LONG = 0x20;
-
- /**
- * The maximum term length (used for byte[] buffer size)
- * for encoding long values.
- * @see #longToPrefixCoded(long,int,BytesRef)
- */
- public static final int BUF_SIZE_LONG = 63/7 + 2;
-
- /**
- * Integers are stored at lower precision by shifting off lower bits. The shift count is
- * stored as SHIFT_START_INT+shift in the first byte
- */
- public static final byte SHIFT_START_INT = 0x60;
-
- /**
- * The maximum term length (used for byte[] buffer size)
- * for encoding int values.
- * @see #intToPrefixCoded(int,int,BytesRef)
- */
- public static final int BUF_SIZE_INT = 31/7 + 2;
-
- /**
- * Returns prefix coded bits after reducing the precision by shift bits.
- * This is method is used by {@link NumericTokenStream}.
- * After encoding, {@code bytes.offset} will always be 0.
- * @param val the numeric value
- * @param shift how many bits to strip from the right
- * @param bytes will contain the encoded value
- * @return the hash code for indexing (TermsHash)
- */
- public static int longToPrefixCoded(final long val, final int shift, final BytesRef bytes) {
- if (shift>63 || shift<0)
- throw new IllegalArgumentException("Illegal shift value, must be 0..63");
- int hash, nChars = (63-shift)/7 + 1;
- bytes.offset = 0;
- bytes.length = nChars+1;
- if (bytes.bytes.length < bytes.length) {
- bytes.grow(NumericUtils.BUF_SIZE_LONG);
- }
- bytes.bytes[0] = (byte) (hash = (SHIFT_START_LONG + shift));
- long sortableBits = val ^ 0x8000000000000000L;
- sortableBits >>>= shift;
- while (nChars > 0) {
- // Store 7 bits per byte for compatibility
- // with UTF-8 encoding of terms
- bytes.bytes[nChars--] = (byte)(sortableBits & 0x7f);
- sortableBits >>>= 7;
- }
- // calculate hash
- for (int i = 1; i < bytes.length; i++) {
- hash = 31*hash + bytes.bytes[i];
- }
- return hash;
- }
-
- /**
- * Returns prefix coded bits after reducing the precision by shift bits.
- * This is method is used by {@link NumericTokenStream}.
- * After encoding, {@code bytes.offset} will always be 0.
- * @param val the numeric value
- * @param shift how many bits to strip from the right
- * @param bytes will contain the encoded value
- * @return the hash code for indexing (TermsHash)
- */
- public static int intToPrefixCoded(final int val, final int shift, final BytesRef bytes) {
- if (shift>31 || shift<0)
- throw new IllegalArgumentException("Illegal shift value, must be 0..31");
- int hash, nChars = (31-shift)/7 + 1;
- bytes.offset = 0;
- bytes.length = nChars+1;
- if (bytes.bytes.length < bytes.length) {
- bytes.grow(NumericUtils.BUF_SIZE_INT);
- }
- bytes.bytes[0] = (byte) (hash = (SHIFT_START_INT + shift));
- int sortableBits = val ^ 0x80000000;
- sortableBits >>>= shift;
- while (nChars > 0) {
- // Store 7 bits per byte for compatibility
- // with UTF-8 encoding of terms
- bytes.bytes[nChars--] = (byte)(sortableBits & 0x7f);
- sortableBits >>>= 7;
- }
- // calculate hash
- for (int i = 1; i < bytes.length; i++) {
- hash = 31*hash + bytes.bytes[i];
- }
- return hash;
- }
-
- /**
- * Returns the shift value from a prefix encoded {@code long}.
- * @throws NumberFormatException if the supplied {@link BytesRef} is
- * not correctly prefix encoded.
- */
- public static int getPrefixCodedLongShift(final BytesRef val) {
- final int shift = val.bytes[val.offset] - SHIFT_START_LONG;
- if (shift > 63 || shift < 0)
- throw new NumberFormatException("Invalid shift value (" + shift + ") in prefixCoded bytes (is encoded value really an INT?)");
- return shift;
- }
-
- /**
- * Returns the shift value from a prefix encoded {@code int}.
- * @throws NumberFormatException if the supplied {@link BytesRef} is
- * not correctly prefix encoded.
- */
- public static int getPrefixCodedIntShift(final BytesRef val) {
- final int shift = val.bytes[val.offset] - SHIFT_START_INT;
- if (shift > 31 || shift < 0)
- throw new NumberFormatException("Invalid shift value in prefixCoded bytes (is encoded value really an INT?)");
- return shift;
- }
-
- /**
- * Returns a long from prefixCoded bytes.
- * Rightmost bits will be zero for lower precision codes.
- * This method can be used to decode a term's value.
- * @throws NumberFormatException if the supplied {@link BytesRef} is
- * not correctly prefix encoded.
- * @see #longToPrefixCoded(long,int,BytesRef)
- */
- public static long prefixCodedToLong(final BytesRef val) {
- long sortableBits = 0L;
- for (int i=val.offset+1, limit=val.offset+val.length; ilong.
- * The value is converted by getting their IEEE 754 floating-point "double format"
- * bit layout and then some bits are swapped, to be able to compare the result as long.
- * By this the precision is not reduced, but the value can easily used as a long.
- * @see #sortableLongToDouble
- */
- public static long doubleToSortableLong(double val) {
- long f = Double.doubleToRawLongBits(val);
- if (f<0) f ^= 0x7fffffffffffffffL;
- return f;
- }
-
- /**
- * Converts a sortable long back to a double.
- * @see #doubleToSortableLong
- */
- public static double sortableLongToDouble(long val) {
- if (val<0) val ^= 0x7fffffffffffffffL;
- return Double.longBitsToDouble(val);
- }
-
- /**
- * Converts a float value to a sortable signed int.
- * The value is converted by getting their IEEE 754 floating-point "float format"
- * bit layout and then some bits are swapped, to be able to compare the result as int.
- * By this the precision is not reduced, but the value can easily used as an int.
- * @see #sortableIntToFloat
- */
- public static int floatToSortableInt(float val) {
- int f = Float.floatToRawIntBits(val);
- if (f<0) f ^= 0x7fffffff;
- return f;
- }
-
- /**
- * Converts a sortable int back to a float.
- * @see #floatToSortableInt
- */
- public static float sortableIntToFloat(int val) {
- if (val<0) val ^= 0x7fffffff;
- return Float.intBitsToFloat(val);
- }
-
- /**
- * Splits a long range recursively.
- * You may implement a builder that adds clauses to a
- * {@link org.apache.lucene.search.BooleanQuery} for each call to its
- * {@link LongRangeBuilder#addRange(BytesRef,BytesRef)}
- * method.
- * Impl to it.
- */
- public static final AttributeFactory DEFAULT_ATTRIBUTE_FACTORY = new DefaultAttributeFactory();
-
- private static final class DefaultAttributeFactory extends AttributeFactory {
- private static final WeakHashMapatt is added to
- * the AttributeSource, because the provided attributes may already exist.
- * You should always retrieve the wanted attributes using {@link #getAttribute} after adding
- * with this method and cast to your class.
- * The recommended way to use custom implementations is using an {@link AttributeFactory}.
- *
- *
- *
- * @see #reflectWith(AttributeReflector)
- */
- public final String reflectAsString(final boolean prependAttClass) {
- final StringBuilder buffer = new StringBuilder();
- reflectWith(new AttributeReflector() {
- public void reflect(Class attClass, String key, Object value) {
- if (buffer.length() > 0) {
- buffer.append(',');
- }
- if (prependAttClass) {
- buffer.append(attClass.getName()).append('#');
- }
- buffer.append(key).append('=').append((value == null) ? "null" : value);
- }
- });
- return buffer.toString();
- }
-
- /**
- * This method is for introspection of attributes, it should simply
- * add the key/values this AttributeSource holds to the given {@link AttributeReflector}.
- *
- *
- *
- *
- * @see #reflectWith(AttributeReflector)
- */
- public final String reflectAsString(final boolean prependAttClass) {
- final StringBuilder buffer = new StringBuilder();
- reflectWith(new AttributeReflector() {
- public void reflect(Class attClass, String key, Object value) {
- if (buffer.length() > 0) {
- buffer.append(',');
- }
- if (prependAttClass) {
- buffer.append(attClass.getName()).append('#');
- }
- buffer.append(key).append('=').append((value == null) ? "null" : value);
- }
- });
- return buffer.toString();
- }
-
- /**
- * This method is for introspection of attributes, it should simply
- * add the key/values this attribute holds to the given {@link AttributeReflector}.
- *
- *
- * public void reflectWith(AttributeReflector reflector) {
- * reflector.reflect(CharTermAttribute.class, "term", term());
- * reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", getPositionIncrement());
- * }
- *
- *
- * int field:
+ *
+ * Field field = new Field(name, new NumericTokenStream(precisionStep).setIntValue(value));
+ * field.setOmitNorms(true);
+ * field.setOmitTermFreqAndPositions(true);
+ * document.add(field);
+ *
+ *
+ *
+ * NumericTokenStream stream = new NumericTokenStream(precisionStep);
+ * Field field = new Field(name, stream);
+ * field.setOmitNorms(true);
+ * field.setOmitTermFreqAndPositions(true);
+ * Document document = new Document();
+ * document.add(field);
+ *
+ * for(all documents) {
+ * stream.setIntValue(value)
+ * writer.addDocument(document);
+ * }
+ *
+ *
+ * NumericTokenStream
+ * instance for each.precisionStep
+ * parameter as well as how numeric fields work under the hood.precisionStep
+ * {@link NumericUtils#PRECISION_STEP_DEFAULT} (4). The stream is not yet initialized,
+ * before using set a value using the various set???Value() methods.
+ */
+ public NumericTokenStream() {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, NumericUtils.PRECISION_STEP_DEFAULT);
+ }
+
+ /**
+ * Creates a token stream for numeric values with the specified
+ * precisionStep. The stream is not yet initialized,
+ * before using set a value using the various set???Value() methods.
+ */
+ public NumericTokenStream(final int precisionStep) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, precisionStep);
+ }
+
+ /**
+ * Expert: Creates a token stream for numeric values with the specified
+ * precisionStep using the given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ * The stream is not yet initialized,
+ * before using set a value using the various set???Value() methods.
+ */
+ public NumericTokenStream(AttributeFactory factory, final int precisionStep) {
+ super(new NumericAttributeFactory(factory));
+ if (precisionStep < 1)
+ throw new IllegalArgumentException("precisionStep must be >=1");
+ this.precisionStep = precisionStep;
+ numericAtt.setShift(-precisionStep);
+ }
+
+ /**
+ * Initializes the token stream with the supplied long value.
+ *
+ * @param value the value, for which this TokenStream should enumerate tokens.
+ * @return this instance, because of this you can use it the following way:
+ * new Field(name, new NumericTokenStream(precisionStep).setLongValue(value))
+ */
+ public NumericTokenStream setLongValue(final long value) {
+ numericAtt.init(value, valSize = 64, precisionStep, -precisionStep);
+ return this;
+ }
+
+ /**
+ * Initializes the token stream with the supplied int value.
+ *
+ * @param value the value, for which this TokenStream should enumerate tokens.
+ * @return this instance, because of this you can use it the following way:
+ * new Field(name, new NumericTokenStream(precisionStep).setIntValue(value))
+ */
+ public NumericTokenStream setIntValue(final int value) {
+ numericAtt.init(value, valSize = 32, precisionStep, -precisionStep);
+ return this;
+ }
+
+ /**
+ * Initializes the token stream with the supplied double value.
+ *
+ * @param value the value, for which this TokenStream should enumerate tokens.
+ * @return this instance, because of this you can use it the following way:
+ * new Field(name, new NumericTokenStream(precisionStep).setDoubleValue(value))
+ */
+ public NumericTokenStream setDoubleValue(final double value) {
+ numericAtt.init(NumericUtils.doubleToSortableLong(value), valSize = 64, precisionStep, -precisionStep);
+ return this;
+ }
+
+ /**
+ * Initializes the token stream with the supplied float value.
+ *
+ * @param value the value, for which this TokenStream should enumerate tokens.
+ * @return this instance, because of this you can use it the following way:
+ * new Field(name, new NumericTokenStream(precisionStep).setFloatValue(value))
+ */
+ public NumericTokenStream setFloatValue(final float value) {
+ numericAtt.init(NumericUtils.floatToSortableInt(value), valSize = 32, precisionStep, -precisionStep);
+ return this;
+ }
+
+ @Override
+ public void reset() {
+ if (valSize == 0)
+ throw new IllegalStateException("call set???Value() before usage");
+ numericAtt.setShift(-precisionStep);
+ }
+
+ @Override
+ public boolean incrementToken() {
+ if (valSize == 0)
+ throw new IllegalStateException("call set???Value() before usage");
+
+ // this will only clear all other attributes in this TokenStream
+ clearAttributes();
+
+ final int shift = numericAtt.incShift();
+ typeAtt.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC : TOKEN_TYPE_LOWER_PREC);
+ posIncrAtt.setPositionIncrement((shift == 0) ? 1 : 0);
+ return (shift < valSize);
+ }
+
+ /**
+ * Returns the precision step.
+ */
+ public int getPrecisionStep() {
+ return precisionStep;
+ }
+
+ // members
+ private final NumericTermAttribute numericAtt = addAttribute(NumericTermAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+
+ private int valSize = 0; // valSize==0 means not initialized
+ private final int precisionStep;
+}
Index: lucene/src/declarations/org/apache/lucene/analysis/CharReader.java
===================================================================
--- lucene/src/declarations/org/apache/lucene/analysis/CharReader.java (revision 0)
+++ lucene/src/declarations/org/apache/lucene/analysis/CharReader.java (revision 0)
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * CharReader is a Reader wrapper. It reads chars from
+ * Reader and outputs {@link CharStream}, defining an
+ * identify function {@link #correctOffset} method that
+ * simply returns the provided offset.
+ */
+public final class CharReader extends CharStream {
+
+ private final Reader input;
+
+ public static CharStream get(Reader input) {
+ return input instanceof CharStream ?
+ (CharStream)input : new CharReader(input);
+ }
+
+ private CharReader(Reader in) {
+ input = in;
+ }
+
+ @Override
+ public int correctOffset(int currentOff) {
+ return currentOff;
+ }
+
+ @Override
+ public void close() throws IOException {
+ input.close();
+ }
+
+ @Override
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ return input.read(cbuf, off, len);
+ }
+
+ @Override
+ public boolean markSupported(){
+ return input.markSupported();
+ }
+
+ @Override
+ public void mark( int readAheadLimit ) throws IOException {
+ input.mark(readAheadLimit);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ input.reset();
+ }
+}
Index: lucene/src/declarations/org/apache/lucene/analysis/Token.java
===================================================================
--- lucene/src/declarations/org/apache/lucene/analysis/Token.java (revision 0)
+++ lucene/src/declarations/org/apache/lucene/analysis/Token.java (revision 0)
@@ -0,0 +1,648 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.index.Payload;
+
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
+
+/**
+ A Token is an occurrence of a term from the text of a field. It consists of
+ a term's text, the start and end offset of the term in the text of the field,
+ and a type string.
+
+
+
+
+
+
+ A few things to note:
+
+
+ return reusableToken.reinit(string, startOffset, endOffset[, type]);
+
+
+
+ return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]);
+
+
+
+ return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]);
+
+
+
+ return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]);
+
+
+
+ return reusableToken.reinit(source.buffer(), 0, source.length(), source.startOffset(), source.endOffset()[, source.type()]);
+
+
+
+ TokenStreams can be chained, one cannot assume that the Token's current type is correct.{@linkplain #toString toString()} method had to be changed to match the
+ {@link CharSequence} interface introduced by the interface {@link org.apache.lucene.analysis.tokenattributes.CharTermAttribute}.
+ This method now only prints the term text, no additional information anymore.
+
+ *
+ *
+ * @param positionIncrement the distance from the prior term
+ * @see org.apache.lucene.index.DocsAndPositionsEnum
+ */
+ public void setPositionIncrement(int positionIncrement) {
+ if (positionIncrement < 0)
+ throw new IllegalArgumentException
+ ("Increment must be zero or greater: " + positionIncrement);
+ this.positionIncrement = positionIncrement;
+ }
+
+ /** Returns the position increment of this Token.
+ * @see #setPositionIncrement
+ */
+ public int getPositionIncrement() {
+ return positionIncrement;
+ }
+
+ /** Returns this Token's starting offset, the position of the first character
+ corresponding to this token in the source text.
+
+ Note that the difference between endOffset() and startOffset() may not be
+ equal to {@link #length}, as the term text may have been altered by a
+ stemmer or some other filter. */
+ public final int startOffset() {
+ return startOffset;
+ }
+
+ /** Set the starting offset.
+ @see #startOffset() */
+ public void setStartOffset(int offset) {
+ this.startOffset = offset;
+ }
+
+ /** Returns this Token's ending offset, one greater than the position of the
+ last character corresponding to this token in the source text. The length
+ of the token in the source text is (endOffset - startOffset). */
+ public final int endOffset() {
+ return endOffset;
+ }
+
+ /** Set the ending offset.
+ @see #endOffset() */
+ public void setEndOffset(int offset) {
+ this.endOffset = offset;
+ }
+
+ /** Set the starting and ending offset.
+ @see #startOffset() and #endOffset()*/
+ public void setOffset(int startOffset, int endOffset) {
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ }
+
+ /** Returns this Token's lexical type. Defaults to "word". */
+ public final String type() {
+ return type;
+ }
+
+ /** Set the lexical type.
+ @see #type() */
+ public final void setType(String type) {
+ this.type = type;
+ }
+
+ /**
+ *
+ *
+ * Get the bitset for any bits that have been set. This is completely distinct from {@link #type()}, although they do share similar purposes.
+ * The flags can be used to encode information about the token for use by other {@link org.apache.lucene.analysis.TokenFilter}s.
+ *
+ *
+ * @return The bits
+ * @lucene.experimental While we think this is here to stay, we may want to change it to be a long.
+ */
+ public int getFlags() {
+ return flags;
+ }
+
+ /**
+ * @see #getFlags()
+ */
+ public void setFlags(int flags) {
+ this.flags = flags;
+ }
+
+ /**
+ * Returns this Token's payload.
+ */
+ public Payload getPayload() {
+ return this.payload;
+ }
+
+ /**
+ * Sets this Token's payload.
+ */
+ public void setPayload(Payload payload) {
+ this.payload = payload;
+ }
+
+ /** Resets the term text, payload, flags, and positionIncrement,
+ * startOffset, endOffset and token type to default.
+ */
+ @Override
+ public void clear() {
+ super.clear();
+ payload = null;
+ positionIncrement = 1;
+ flags = 0;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
+
+ @Override
+ public Object clone() {
+ Token t = (Token)super.clone();
+ // Do a deep clone
+ if (payload != null) {
+ t.payload = (Payload) payload.clone();
+ }
+ return t;
+ }
+
+ /** Makes a clone, but replaces the term buffer &
+ * start/end offset in the process. This is more
+ * efficient than doing a full clone (and then calling
+ * {@link #copyBuffer}) because it saves a wasted copy of the old
+ * termBuffer. */
+ public Token clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
+ final Token t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset);
+ t.positionIncrement = positionIncrement;
+ t.flags = flags;
+ t.type = type;
+ if (payload != null)
+ t.payload = (Payload) payload.clone();
+ return t;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+
+ if (obj instanceof Token) {
+ final Token other = (Token) obj;
+ return (startOffset == other.startOffset &&
+ endOffset == other.endOffset &&
+ flags == other.flags &&
+ positionIncrement == other.positionIncrement &&
+ (type == null ? other.type == null : type.equals(other.type)) &&
+ (payload == null ? other.payload == null : payload.equals(other.payload)) &&
+ super.equals(obj)
+ );
+ } else
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ int code = super.hashCode();
+ code = code * 31 + startOffset;
+ code = code * 31 + endOffset;
+ code = code * 31 + flags;
+ code = code * 31 + positionIncrement;
+ if (type != null)
+ code = code * 31 + type.hashCode();
+ if (payload != null)
+ code = code * 31 + payload.hashCode();
+ return code;
+ }
+
+ // like clear() but doesn't clear termBuffer/text
+ private void clearNoTermBuffer() {
+ payload = null;
+ positionIncrement = 1;
+ flags = 0;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #copyBuffer(char[], int, int)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset},
+ * {@link #setType}
+ * @return this Token instance */
+ public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) {
+ clearNoTermBuffer();
+ copyBuffer(newTermBuffer, newTermOffset, newTermLength);
+ payload = null;
+ positionIncrement = 1;
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #copyBuffer(char[], int, int)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType} on Token.DEFAULT_TYPE
+ * @return this Token instance */
+ public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
+ clearNoTermBuffer();
+ copyBuffer(newTermBuffer, newTermOffset, newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType}
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newStartOffset, int newEndOffset, String newType) {
+ clear();
+ append(newTerm);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence, int, int)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType}
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) {
+ clear();
+ append(newTerm, newTermOffset, newTermOffset + newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType} on Token.DEFAULT_TYPE
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newStartOffset, int newEndOffset) {
+ clear();
+ append(newTerm);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence, int, int)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType} on Token.DEFAULT_TYPE
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
+ clear();
+ append(newTerm, newTermOffset, newTermOffset + newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /**
+ * Copy the prototype token's fields into this one. Note: Payloads are shared.
+ * @param prototype
+ */
+ public void reinit(Token prototype) {
+ copyBuffer(prototype.buffer(), 0, prototype.length());
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ /**
+ * Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.
+ * @param prototype
+ * @param newTerm
+ */
+ public void reinit(Token prototype, String newTerm) {
+ setEmpty().append(newTerm);
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ /**
+ * Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.
+ * @param prototype
+ * @param newTermBuffer
+ * @param offset
+ * @param length
+ */
+ public void reinit(Token prototype, char[] newTermBuffer, int offset, int length) {
+ copyBuffer(newTermBuffer, offset, length);
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ if (target instanceof Token) {
+ final Token to = (Token) target;
+ to.reinit(this);
+ // reinit shares the payload, so clone it:
+ if (payload !=null) {
+ to.payload = (Payload) payload.clone();
+ }
+ } else {
+ super.copyTo(target);
+ ((OffsetAttribute) target).setOffset(startOffset, endOffset);
+ ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement);
+ ((PayloadAttribute) target).setPayload((payload == null) ? null : (Payload) payload.clone());
+ ((FlagsAttribute) target).setFlags(flags);
+ ((TypeAttribute) target).setType(type);
+ }
+ }
+
+ @Override
+ public void reflectWith(AttributeReflector reflector) {
+ super.reflectWith(reflector);
+ reflector.reflect(OffsetAttribute.class, "startOffset", startOffset);
+ reflector.reflect(OffsetAttribute.class, "endOffset", endOffset);
+ reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", positionIncrement);
+ reflector.reflect(PayloadAttribute.class, "payload", payload);
+ reflector.reflect(FlagsAttribute.class, "flags", flags);
+ reflector.reflect(TypeAttribute.class, "type", type);
+ }
+
+ /** Convenience factory that returns Token as implementation for the basic
+ * attributes and return the default impl (with "Impl" appended) for all other
+ * attributes.
+ * @since 3.0
+ */
+ public static final AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
+ new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
+
+ /** Expert: Creates a TokenAttributeFactory returning {@link Token} as instance for the basic attributes
+ * and for all other attributes calls the given delegate factory.
+ * @since 3.0
+ */
+ public static final class TokenAttributeFactory extends AttributeSource.AttributeFactory {
+
+ private final AttributeSource.AttributeFactory delegate;
+
+ /** Expert: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
+ * and for all other attributes calls the given delegate factory. */
+ public TokenAttributeFactory(AttributeSource.AttributeFactory delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public AttributeImpl createAttributeInstance(Class attClass) {
+ return attClass.isAssignableFrom(Token.class)
+ ? new Token() : delegate.createAttributeInstance(attClass);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) return true;
+ if (other instanceof TokenAttributeFactory) {
+ final TokenAttributeFactory af = (TokenAttributeFactory) other;
+ return this.delegate.equals(af.delegate);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return delegate.hashCode() ^ 0x0a45aa31;
+ }
+ }
+
+}
Index: lucene/src/declarations/org/apache/lucene/analysis/CachingTokenFilter.java
===================================================================
--- lucene/src/declarations/org/apache/lucene/analysis/CachingTokenFilter.java (revision 0)
+++ lucene/src/declarations/org/apache/lucene/analysis/CachingTokenFilter.java (revision 0)
@@ -0,0 +1,86 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.lucene.util.AttributeSource;
+
+/**
+ * This class can be used if the token attributes of a TokenStream
+ * are intended to be consumed more than once. It caches
+ * all token attribute states locally in a List.
+ *
+ * Hints, Tips and Traps
+
+
+
+
+Invoking the Analyzer
+
+
+ However an application might invoke Analysis of any text for testing or for any other purpose, something like:
+
+ Analyzer analyzer = new StandardAnalyzer(); // or any other analyzer
+ TokenStream ts = analyzer.tokenStream("myfield",new StringReader("some text goes here"));
+ while (ts.incrementToken()) {
+ System.out.println("token: "+ts));
+ }
+
+Indexing Analysis vs. Search Analysis
+
+
+
+
+ This might sometimes require a modified analyzer – see the next section on how to do that.
+
Creating your own Analyzer is straightforward. It usually involves either wrapping an existing Tokenizer and set of TokenFilters to create a new Analyzer +or creating both the Analyzer and a Tokenizer or TokenFilter. Before pursuing this approach, you may find it worthwhile +to explore the modules/analysis library and/or ask on the java-user@lucene.apache.org mailing list first to see if what you need already exists. +If you are still committed to creating your own Analyzer or TokenStream derivation (Tokenizer or TokenFilter) have a look at +the source code of any one of the many samples located in this package. +
++ The following sections discuss some aspects of implementing your own analyzer. +
++ When {@link org.apache.lucene.document.Document#add(org.apache.lucene.document.Fieldable) document.add(field)} + is called multiple times for the same field name, we could say that each such call creates a new + section for that field in that document. + In fact, a separate call to + {@link org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String, java.io.Reader) tokenStream(field,reader)} + would take place for each of these so called "sections". + However, the default Analyzer behavior is to treat all these sections as one large section. + This allows phrase search and proximity search to seamlessly cross + boundaries between these "sections". + In other words, if a certain field "f" is added like this: +
+ document.add(new Field("f","first ends",...);
+ document.add(new Field("f","starts two",...);
+ indexWriter.addDocument(document);
+
+ Then, a phrase search for "ends starts" would find that document.
+ Where desired, this behavior can be modified by introducing a "position gap" between consecutive field "sections",
+ simply by overriding
+ {@link org.apache.lucene.analysis.Analyzer#getPositionIncrementGap(java.lang.String) Analyzer.getPositionIncrementGap(fieldName)}:
+
+ Analyzer myAnalyzer = new StandardAnalyzer() {
+ public int getPositionIncrementGap(String fieldName) {
+ return 10;
+ }
+ };
+
+
++ By default, all tokens created by Analyzers and Tokenizers have a + {@link org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute#getPositionIncrement() position increment} of one. + This means that the position stored for that token in the index would be one more than + that of the previous token. + Recall that phrase and proximity searches rely on position info. +
++ If the selected analyzer filters the stop words "is" and "the", then for a document + containing the string "blue is the sky", only the tokens "blue", "sky" are indexed, + with position("sky") = 1 + position("blue"). Now, a phrase query "blue is the sky" + would find that document, because the same analyzer filters the same stop words from + that query. But also the phrase query "blue sky" would find that document. +
++ If this behavior does not fit the application needs, + a modified analyzer can be used, that would increment further the positions of + tokens following a removed stop word, using + {@link org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute#setPositionIncrement(int)}. + This can be done with something like: +
+ public TokenStream tokenStream(final String fieldName, Reader reader) {
+ final TokenStream ts = someAnalyzer.tokenStream(fieldName, reader);
+ TokenStream res = new TokenStream() {
+ CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+
+ public boolean incrementToken() throws IOException {
+ int extraIncrement = 0;
+ while (true) {
+ boolean hasNext = ts.incrementToken();
+ if (hasNext) {
+ if (stopWords.contains(termAtt.toString())) {
+ extraIncrement++; // filter this word
+ continue;
+ }
+ if (extraIncrement>0) {
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement()+extraIncrement);
+ }
+ }
+ return hasNext;
+ }
+ }
+ };
+ return res;
+ }
+
+ Now, with this modified analyzer, the phrase query "blue sky" would find that document.
+ But note that this is yet not a perfect solution, because any phrase query "blue w1 w2 sky"
+ where both w1 and w2 are stop words would match that document.
+
++ Few more use cases for modifying position increments are: +
+ With Lucene 2.9 we introduce a new TokenStream API. The old API used to produce Tokens. A Token + has getter and setter methods for different properties like positionIncrement and termText. + While this approach was sufficient for the default indexing format, it is not versatile enough for + Flexible Indexing, a term which summarizes the effort of making the Lucene indexer pluggable and extensible for custom + index formats. +
++A fully customizable indexer means that users will be able to store custom data structures on disk. Therefore an API +is necessary that can transport custom types of data from the documents to the indexer. +
++ Lucene now provides six Attributes out of the box, which replace the variables the Token class has: +
The term text of a token.
The start and end offset of token in characters.
See above for detailed information about position increment.
The payload that a Token can optionally have.
The type of the token. Default is 'word'.
Optional flags a token can have.
Class)
+of an Attribute as an argument and returns an instance. If an Attribute of the same type was previously added, then
+the already existing instance is returned, otherwise a new instance is created and returned. Therefore TokenStreams/-Filters
+can safely call addAttribute() with the same Attribute type multiple times. Even consumers of TokenStreams should
+normally call addAttribute() instead of getAttribute(), because it would not fail if the TokenStream does not have this
+Attribute (getAttribute() would throw an IllegalArgumentException, if the Attribute is missing). More advanced code
+could simply check with hasAttribute(), if a TokenStream has it, and may conditionally leave out processing for
+extra performance.
+
+public class MyAnalyzer extends Analyzer {
+
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ TokenStream stream = new WhitespaceTokenizer(reader);
+ return stream;
+ }
+
+ public static void main(String[] args) throws IOException {
+ // text to tokenize
+ final String text = "This is a demo of the new TokenStream API";
+
+ MyAnalyzer analyzer = new MyAnalyzer();
+ TokenStream stream = analyzer.tokenStream("field", new StringReader(text));
+
+ // get the CharTermAttribute from the TokenStream
+ CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
+
+ stream.reset();
+
+ // print all tokens until stream is exhausted
+ while (stream.incrementToken()) {
+ System.out.println(termAtt.toString());
+ }
+
+ stream.end()
+ stream.close();
+ }
+}
+
+In this easy example a simple white space tokenization is performed. In main() a loop consumes the stream and
+prints the term text of the tokens by accessing the CharTermAttribute that the WhitespaceTokenizer provides.
+Here is the output:
++This +is +a +demo +of +the +new +TokenStream +API ++
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ TokenStream stream = new WhitespaceTokenizer(reader);
+ stream = new LengthFilter(stream, 3, Integer.MAX_VALUE);
+ return stream;
+ }
+
+Note how now only words with 3 or more characters are contained in the output:
++This +demo +the +new +TokenStream +API ++Now let's take a look how the LengthFilter is implemented (it is part of Lucene's core): +
+public final class LengthFilter extends TokenFilter {
+
+ final int min;
+ final int max;
+
+ private CharTermAttribute termAtt;
+
+ /**
+ * Build a filter that removes words that are too long or too
+ * short from the text.
+ */
+ public LengthFilter(TokenStream in, int min, int max)
+ {
+ super(in);
+ this.min = min;
+ this.max = max;
+ termAtt = addAttribute(CharTermAttribute.class);
+ }
+
+ /**
+ * Returns the next input Token whose term() is the right len
+ */
+ public final boolean incrementToken() throws IOException
+ {
+ assert termAtt != null;
+ // return the first non-stop word found
+ while (input.incrementToken()) {
+ int len = termAtt.length();
+ if (len >= min && len <= max) {
+ return true;
+ }
+ // note: else we ignore it but should we index each part of it?
+ }
+ // reached EOS -- return null
+ return false;
+ }
+}
+
+The CharTermAttribute is added in the constructor and stored in the instance variable termAtt.
+Remember that there can only be a single instance of CharTermAttribute in the chain, so in our example the
+addAttribute() call in LengthFilter returns the TermAttribute that the WhitespaceTokenizer already added. The tokens
+are retrieved from the input stream in the incrementToken() method. By looking at the term text
+in the CharTermAttribute the length of the term can be determined and too short or too long tokens are skipped.
+Note how incrementToken() can efficiently access the instance variable; no attribute lookup
+is neccessary. The same is true for the consumer, which can simply use local references to the Attributes.
+
+PartOfSpeechAttribute. First we need to define the interface of the new Attribute:
+
+ public interface PartOfSpeechAttribute extends Attribute {
+ public static enum PartOfSpeech {
+ Noun, Verb, Adjective, Adverb, Pronoun, Preposition, Conjunction, Article, Unknown
+ }
+
+ public void setPartOfSpeech(PartOfSpeech pos);
+
+ public PartOfSpeech getPartOfSpeech();
+ }
+
+
+Now we also need to write the implementing class. The name of that class is important here: By default, Lucene
+checks if there is a class with the name of the Attribute with the postfix 'Impl'. In this example, we would
+consequently call the implementing class PartOfSpeechAttributeImpl.
+public final class PartOfSpeechAttributeImpl extends AttributeImpl
+ implements PartOfSpeechAttribute{
+
+ private PartOfSpeech pos = PartOfSpeech.Unknown;
+
+ public void setPartOfSpeech(PartOfSpeech pos) {
+ this.pos = pos;
+ }
+
+ public PartOfSpeech getPartOfSpeech() {
+ return pos;
+ }
+
+ public void clear() {
+ pos = PartOfSpeech.Unknown;
+ }
+
+ public void copyTo(AttributeImpl target) {
+ ((PartOfSpeechAttributeImpl) target).pos = pos;
+ }
+
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+
+ if (other instanceof PartOfSpeechAttributeImpl) {
+ return pos == ((PartOfSpeechAttributeImpl) other).pos;
+ }
+
+ return false;
+ }
+
+ public int hashCode() {
+ return pos.ordinal();
+ }
+}
+
+This is a simple Attribute implementation has only a single variable that stores the part-of-speech of a token. It extends the
+new AttributeImpl class and therefore implements its abstract methods clear(), copyTo(), equals(), hashCode().
+Now we need a TokenFilter that can set this new PartOfSpeechAttribute for each token. In this example we show a very naive filter
+that tags every word with a leading upper-case letter as a 'Noun' and all other words as 'Unknown'.
+
+ public static class PartOfSpeechTaggingFilter extends TokenFilter {
+ PartOfSpeechAttribute posAtt;
+ CharTermAttribute termAtt;
+
+ protected PartOfSpeechTaggingFilter(TokenStream input) {
+ super(input);
+ posAtt = addAttribute(PartOfSpeechAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
+ }
+
+ public boolean incrementToken() throws IOException {
+ if (!input.incrementToken()) {return false;}
+ posAtt.setPartOfSpeech(determinePOS(termAtt.buffer(), 0, termAtt.length()));
+ return true;
+ }
+
+ // determine the part of speech for the given term
+ protected PartOfSpeech determinePOS(char[] term, int offset, int length) {
+ // naive implementation that tags every uppercased word as noun
+ if (length > 0 && Character.isUpperCase(term[0])) {
+ return PartOfSpeech.Noun;
+ }
+ return PartOfSpeech.Unknown;
+ }
+ }
+
+Just like the LengthFilter, this new filter accesses the attributes it needs in the constructor and
+stores references in instance variables. Notice how you only need to pass in the interface of the new
+Attribute and instantiating the correct class is automatically been taken care of.
+Now we need to add the filter to the chain:
+
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ TokenStream stream = new WhitespaceTokenizer(reader);
+ stream = new LengthFilter(stream, 3, Integer.MAX_VALUE);
+ stream = new PartOfSpeechTaggingFilter(stream);
+ return stream;
+ }
+
+Now let's look at the output:
++This +demo +the +new +TokenStream +API ++Apparently it hasn't changed, which shows that adding a custom attribute to a TokenStream/Filter chain does not +affect any existing consumers, simply because they don't know the new Attribute. Now let's change the consumer +to make use of the new PartOfSpeechAttribute and print it out: +
+ public static void main(String[] args) throws IOException {
+ // text to tokenize
+ final String text = "This is a demo of the new TokenStream API";
+
+ MyAnalyzer analyzer = new MyAnalyzer();
+ TokenStream stream = analyzer.tokenStream("field", new StringReader(text));
+
+ // get the CharTermAttribute from the TokenStream
+ CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
+
+ // get the PartOfSpeechAttribute from the TokenStream
+ PartOfSpeechAttribute posAtt = stream.addAttribute(PartOfSpeechAttribute.class);
+
+ stream.reset();
+
+ // print all tokens until stream is exhausted
+ while (stream.incrementToken()) {
+ System.out.println(termAtt.toString() + ": " + posAtt.getPartOfSpeech());
+ }
+
+ stream.end();
+ stream.close();
+ }
+
+The change that was made is to get the PartOfSpeechAttribute from the TokenStream and print out its contents in
+the while loop that consumes the stream. Here is the new output:
++This: Noun +demo: Unknown +the: Unknown +new: Unknown +TokenStream: Noun +API: Noun ++Each word is now followed by its assigned PartOfSpeech tag. Of course this is a naive +part-of-speech tagging. The word 'This' should not even be tagged as noun; it is only spelled capitalized because it +is the first word of a sentence. Actually this is a good opportunity for an excerise. To practice the usage of the new +API the reader could now write an Attribute and TokenFilter that can specify for each word if it was the first token +of a sentence or not. Then the PartOfSpeechTaggingFilter can make use of this knowledge and only tag capitalized words +as nouns if not the first word of a sentence (we know, this is still not a correct behavior, but hey, it's a good exercise). +As a small hint, this is how the new Attribute class could begin: +
+ public class FirstTokenOfSentenceAttributeImpl extends Attribute
+ implements FirstTokenOfSentenceAttribute {
+
+ private boolean firstToken;
+
+ public void setFirstToken(boolean firstToken) {
+ this.firstToken = firstToken;
+ }
+
+ public boolean getFirstToken() {
+ return firstToken;
+ }
+
+ public void clear() {
+ firstToken = false;
+ }
+
+ ...
+
+
+
Property changes on: lucene/src/declarations/org/apache/lucene/LucenePackage.java
___________________________________________________________________
Added: svn:eol-style
+ native
Property changes on: lucene/src/declarations/org/apache/lucene/index/Payload.java
___________________________________________________________________
Added: svn:eol-style
+ native
Index: lucene/src/declarations/org/apache/lucene/util/Attribute.java
===================================================================
--- lucene/src/declarations/org/apache/lucene/util/Attribute.java (revision 0)
+++ lucene/src/declarations/org/apache/lucene/util/Attribute.java (revision 0)
@@ -0,0 +1,24 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Base interface for attributes.
+ */
+public interface Attribute {
+}
Index: lucene/src/declarations/org/apache/lucene/util/AttributeReflector.java
===================================================================
--- lucene/src/declarations/org/apache/lucene/util/AttributeReflector.java (revision 0)
+++ lucene/src/declarations/org/apache/lucene/util/AttributeReflector.java (revision 0)
@@ -0,0 +1,34 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This interface is used to reflect contents of {@link AttributeSource} or {@link AttributeImpl}.
+ */
+public interface AttributeReflector {
+
+ /**
+ * This method gets called for every property in an {@link AttributeImpl}/{@link AttributeSource}
+ * passing the class name of the {@link Attribute}, a key and the actual value.
+ * E.g., an invocation of {@link org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl#reflectWith}
+ * would call this method once using {@code org.apache.lucene.analysis.tokenattributes.CharTermAttribute.class}
+ * as attribute class, {@code "term"} as key and the actual value as a String.
+ */
+ public void reflect(Class attClass, String key, Object value);
+
+}
Property changes on: lucene/src/declarations/org/apache/lucene/util/IntsRef.java
___________________________________________________________________
Added: svn:eol-style
+ native
Property changes on: lucene/src/declarations/org/apache/lucene/util/SorterTemplate.java
___________________________________________________________________
Added: svn:keywords
+ Date Author Id Revision HeadURL
Added: svn:eol-style
+ native
Property changes on: lucene/src/declarations/org/apache/lucene/util/Constants.java
___________________________________________________________________
Added: cvs2svn:cvs-rev
+ 1.3
Added: svn:keywords
+ Author Date Id Revision
Added: svn:eol-style
+ native
Property changes on: lucene/src/declarations/org/apache/lucene/util/ArrayUtil.java
___________________________________________________________________
Added: svn:eol-style
+ native
Index: lucene/src/declarations/org/apache/lucene/util/NumericUtils.java
===================================================================
--- lucene/src/declarations/org/apache/lucene/util/NumericUtils.java (revision 0)
+++ lucene/src/declarations/org/apache/lucene/util/NumericUtils.java (working copy)
@@ -18,9 +18,6 @@
*/
import org.apache.lucene.analysis.NumericTokenStream;
-import org.apache.lucene.document.NumericField;
-import org.apache.lucene.search.NumericRangeFilter;
-import org.apache.lucene.search.NumericRangeQuery; // for javadocs
/**
* This is a helper class to generate prefix-encoded representations for numerical values
@@ -48,7 +45,7 @@
* For easy usage, the trie algorithm is implemented for indexing inside
* {@link NumericTokenStream} that can index int, long,
* float, and double. For querying,
- * {@link NumericRangeQuery} and {@link NumericRangeFilter} implement the query part
+ * NumericRangeQuery and NumericRangeFilter implement the query part
* for the same data types.
*
*
This class can also be used, to generate lexicographically sortable (according to @@ -63,8 +60,8 @@ private NumericUtils() {} // no instance! /** - * The default precision step used by {@link NumericField}, {@link NumericTokenStream}, - * {@link NumericRangeQuery}, and {@link NumericRangeFilter} as default + * The default precision step used by NumericField, {@link NumericTokenStream}, + * NumericRangeQuery, and NumericRangeFilter as default */ public static final int PRECISION_STEP_DEFAULT = 4; @@ -284,7 +281,7 @@ * {@link org.apache.lucene.search.BooleanQuery} for each call to its * {@link LongRangeBuilder#addRange(BytesRef,BytesRef)} * method. - *
This method is used by {@link NumericRangeQuery}. + *
This method is used by NumericRangeQuery in Lucene core. */ public static void splitLongRange(final LongRangeBuilder builder, final int precisionStep, final long minBound, final long maxBound @@ -298,7 +295,7 @@ * {@link org.apache.lucene.search.BooleanQuery} for each call to its * {@link IntRangeBuilder#addRange(BytesRef,BytesRef)} * method. - *
This method is used by {@link NumericRangeQuery}. + *
This method is used by NumericRangeQuery in Lucene core.
*/
public static void splitIntRange(final IntRangeBuilder builder,
final int precisionStep, final int minBound, final int maxBound
Property changes on: lucene/src/declarations/org/apache/lucene/util/NumericUtils.java
___________________________________________________________________
Added: svn:eol-style
+ native
Property changes on: lucene/src/declarations/org/apache/lucene/util/BytesRef.java
___________________________________________________________________
Added: svn:eol-style
+ native
Index: lucene/src/declarations/org/apache/lucene/util/AttributeSource.java
===================================================================
--- lucene/src/declarations/org/apache/lucene/util/AttributeSource.java (revision 0)
+++ lucene/src/declarations/org/apache/lucene/util/AttributeSource.java (revision 0)
@@ -0,0 +1,504 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.ref.WeakReference;
+import java.util.Collections;
+import java.util.NoSuchElementException;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.WeakHashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.lucene.analysis.TokenStream; // for javadocs
+
+/**
+ * An AttributeSource contains a list of different {@link AttributeImpl}s,
+ * and methods to add and get them. There can only be a single instance
+ * of an attribute in the same AttributeSource instance. This is ensured
+ * by passing in the actual type of the Attribute (Class<Attribute>) to
+ * the {@link #addAttribute(Class)}, which then checks if an instance of
+ * that type is already present. If yes, it returns the instance, otherwise
+ * it creates a new instance and returns it.
+ */
+public class AttributeSource {
+ /**
+ * An AttributeFactory creates instances of {@link AttributeImpl}s.
+ */
+ public static abstract class AttributeFactory {
+ /**
+ * returns an {@link AttributeImpl} for the supplied {@link Attribute} interface class.
+ */
+ public abstract AttributeImpl createAttributeInstance(Class attClass);
+
+ /**
+ * This is the default factory that creates {@link AttributeImpl}s using the
+ * class name of the supplied {@link Attribute} interface class by appending Please note: It is not guaranteed, that
+ * Note that this method does not affect attributes of the targetStream
+ * that are not contained in this state. In other words, if for example
+ * the targetStream contains an OffsetAttribute, but this state doesn't, then
+ * the value of the OffsetAttribute remains unchanged. It might be desirable to
+ * reset its value to the default, in which case the caller should first
+ * call {@link TokenStream#clearAttributes()} on the targetStream.
+ */
+ public final void restoreState(State state) {
+ if (state == null) return;
+
+ do {
+ AttributeImpl targetImpl = attributeImpls.get(state.attribute.getClass());
+ if (targetImpl == null) {
+ throw new IllegalArgumentException("State contains AttributeImpl of type " +
+ state.attribute.getClass().getName() + " that is not in in this AttributeSource");
+ }
+ state.attribute.copyTo(targetImpl);
+ state = state.next;
+ } while (state != null);
+ }
+
+ @Override
+ public int hashCode() {
+ int code = 0;
+ for (State state = getCurrentState(); state != null; state = state.next) {
+ code = code * 31 + state.attribute.hashCode();
+ }
+ return code;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj instanceof AttributeSource) {
+ AttributeSource other = (AttributeSource) obj;
+
+ if (hasAttributes()) {
+ if (!other.hasAttributes()) {
+ return false;
+ }
+
+ if (this.attributeImpls.size() != other.attributeImpls.size()) {
+ return false;
+ }
+
+ // it is only equal if all attribute impls are the same in the same order
+ State thisState = this.getCurrentState();
+ State otherState = other.getCurrentState();
+ while (thisState != null && otherState != null) {
+ if (otherState.attribute.getClass() != thisState.attribute.getClass() || !otherState.attribute.equals(thisState.attribute)) {
+ return false;
+ }
+ thisState = thisState.next;
+ otherState = otherState.next;
+ }
+ return true;
+ } else {
+ return !other.hasAttributes();
+ }
+ } else
+ return false;
+ }
+
+ /**
+ * This method returns the current attribute values as a string in the following format
+ * by calling the {@link #reflectWith(AttributeReflector)} method:
+ *
+ * This method iterates over all Attribute implementations and calls the
+ * corresponding {@link AttributeImpl#reflectWith} method.
+ * Attributes are used to add data in a dynamic, yet type-safe way to a source
+ * of usually streamed objects, e. g. a {@link org.apache.lucene.analysis.TokenStream}.
+ */
+public abstract class AttributeImpl implements Cloneable, Attribute {
+ /**
+ * Clears the values in this AttributeImpl and resets it to its
+ * default value. If this implementation implements more than one Attribute interface
+ * it clears all.
+ */
+ public abstract void clear();
+
+ /**
+ * This method returns the current attribute values as a string in the following format
+ * by calling the {@link #reflectWith(AttributeReflector)} method:
+ *
+ * The default implementation calls {@link AttributeReflector#reflect} for all
+ * non-static fields from the implementing class, using the field name as key
+ * and the field value as value. The Attribute class is also determined by reflection.
+ * Please note that the default implementation can only handle single-Attribute
+ * implementations.
+ *
+ * Custom implementations look like this (e.g. for a combined attribute implementation):
+ * If you implement this method, make sure that for each invocation, the same set of {@link Attribute}
+ * interfaces and keys are passed to {@link AttributeReflector#reflect} in the same order, but possibly
+ * different values. So don't automatically exclude e.g. {@code null} properties!
+ *
+ * @see #reflectAsString(boolean)
+ */
+ public void reflectWith(AttributeReflector reflector) {
+ final Class clazz = this.getClass();
+ final LinkedListImpl to it.
+ */
+ public static final AttributeFactory DEFAULT_ATTRIBUTE_FACTORY = new DefaultAttributeFactory();
+
+ private static final class DefaultAttributeFactory extends AttributeFactory {
+ private static final WeakHashMapatt is added to
+ * the AttributeSource, because the provided attributes may already exist.
+ * You should always retrieve the wanted attributes using {@link #getAttribute} after adding
+ * with this method and cast to your class.
+ * The recommended way to use custom implementations is using an {@link AttributeFactory}.
+ *
+ *
+ *
+ * @see #reflectWith(AttributeReflector)
+ */
+ public final String reflectAsString(final boolean prependAttClass) {
+ final StringBuilder buffer = new StringBuilder();
+ reflectWith(new AttributeReflector() {
+ public void reflect(Class attClass, String key, Object value) {
+ if (buffer.length() > 0) {
+ buffer.append(',');
+ }
+ if (prependAttClass) {
+ buffer.append(attClass.getName()).append('#');
+ }
+ buffer.append(key).append('=').append((value == null) ? "null" : value);
+ }
+ });
+ return buffer.toString();
+ }
+
+ /**
+ * This method is for introspection of attributes, it should simply
+ * add the key/values this AttributeSource holds to the given {@link AttributeReflector}.
+ *
+ *
+ *
+ *
+ * @see #reflectWith(AttributeReflector)
+ */
+ public final String reflectAsString(final boolean prependAttClass) {
+ final StringBuilder buffer = new StringBuilder();
+ reflectWith(new AttributeReflector() {
+ public void reflect(Class attClass, String key, Object value) {
+ if (buffer.length() > 0) {
+ buffer.append(',');
+ }
+ if (prependAttClass) {
+ buffer.append(attClass.getName()).append('#');
+ }
+ buffer.append(key).append('=').append((value == null) ? "null" : value);
+ }
+ });
+ return buffer.toString();
+ }
+
+ /**
+ * This method is for introspection of attributes, it should simply
+ * add the key/values this attribute holds to the given {@link AttributeReflector}.
+ *
+ *
+ * public void reflectWith(AttributeReflector reflector) {
+ * reflector.reflect(CharTermAttribute.class, "term", term());
+ * reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", getPositionIncrement());
+ * }
+ *
+ *
+ *