This filter handles position increments > 1 by inserting filler tokens * (tokens with termtext "_"). It does not handle a position increment of 0. */ -public class ShingleFilter extends TokenFilter { +public final class ShingleFilter extends TokenFilter { private LinkedList shingleBuf = new LinkedList(); private StringBuilder[] shingles; @@ -327,19 +327,7 @@ return true; } - - /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should - * not be overridden. Delegates to the backwards compatibility layer. */ - public final Token next(final Token reusableToken) throws java.io.IOException { - return super.next(reusableToken); - } - /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should - * not be overridden. Delegates to the backwards compatibility layer. */ - public final Token next() throws java.io.IOException { - return super.next(); - } - public void reset() throws IOException { super.reset(); nextToken = null; Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java (revision 824110) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java (working copy) @@ -114,7 +114,7 @@ * NOTE: This filter might not behave correctly if used with custom Attributes, i.e. Attributes other than * the ones located in org.apache.lucene.analysis.tokenattributes. */ -public class ShingleMatrixFilter extends TokenStream { +public final class ShingleMatrixFilter extends TokenStream { public static Character defaultSpacerCharacter = new Character('_'); public static TokenSettingsCodec defaultSettingsCodec = new OneDimensionalNonWeightedTokenSettingsCodec(); @@ -393,18 +393,17 @@ return token; } - /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should - * not be overridden. Delegates to the backwards compatibility layer. */ - public final Token next(final Token reusableToken) throws java.io.IOException { - return super.next(reusableToken); + private Token getNextToken(Token token) throws IOException { + if (!this.incrementToken()) return null; + token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength()); + token.setPositionIncrement(posIncrAtt.getPositionIncrement()); + token.setFlags(flagsAtt.getFlags()); + token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); + token.setType(typeAtt.type()); + token.setPayload(payloadAtt.getPayload()); + return token; } - /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should - * not be overridden. Delegates to the backwards compatibility layer. */ - public final Token next() throws java.io.IOException { - return super.next(); - } - private static final Token request_next_token = new Token(); /** @@ -429,7 +428,7 @@ if (ignoringSinglePrefixOrSuffixShingle && currentShingleLength == 1 && ((currentPermutationRows.get(currentPermutationTokensStartOffset)).getColumn().isFirst() || (currentPermutationRows.get(currentPermutationTokensStartOffset)).getColumn().isLast())) { - return next(reusableToken); + return getNextToken(reusableToken); } int termLength = 0; Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizer.java (revision 824110) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizer.java (working copy) @@ -1,88 +0,0 @@ -package org.apache.lucene.analysis.sinks; -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.analysis.SinkTokenizer; -import org.apache.lucene.analysis.TeeSinkTokenFilter; -import org.apache.lucene.analysis.Token; - -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.text.ParseException; -import java.util.List; -import java.util.Date; - - -/** - * Attempts to parse the {@link org.apache.lucene.analysis.Token#termBuffer()} as a Date using a {@link java.text.DateFormat}. - * If the value is a Date, it will add it to the sink. - *
- * Also marks the sink token with {@link org.apache.lucene.analysis.Token#type()} equal to {@link #DATE_TYPE} - * - * @deprecated Use {@link DateRecognizerSinkFilter} and {@link TeeSinkTokenFilter} instead. - **/ -public class DateRecognizerSinkTokenizer extends SinkTokenizer { - public static final String DATE_TYPE = "date"; - - protected DateFormat dateFormat; - - /** - * Uses {@link java.text.SimpleDateFormat#getDateInstance()} as the {@link java.text.DateFormat} object. - */ - public DateRecognizerSinkTokenizer() { - this(null, SimpleDateFormat.getDateInstance()); - } - - public DateRecognizerSinkTokenizer(DateFormat dateFormat) { - this(null, dateFormat); - } - - /** - * Uses {@link java.text.SimpleDateFormat#getDateInstance()} as the {@link java.text.DateFormat} object. - * @param input The input list of Tokens that are already Dates. They should be marked as type {@link #DATE_TYPE} for completeness - */ - public DateRecognizerSinkTokenizer(List/*typeToMatch then
- * add it to the sink
- *
- * @deprecated Use {@link TokenTypeSinkFilter} and {@link TeeSinkTokenFilter} instead.
- **/
-public class TokenTypeSinkTokenizer extends SinkTokenizer {
-
- private String typeToMatch;
-
- public TokenTypeSinkTokenizer(String typeToMatch) {
- this.typeToMatch = typeToMatch;
- }
-
- public TokenTypeSinkTokenizer(int initCap, String typeToMatch) {
- super(initCap);
- this.typeToMatch = typeToMatch;
- }
-
- public TokenTypeSinkTokenizer(List/*OR_OPERATOR) terms without any modifiers
* are considered optional: for example capital of Hungary is equal to
* capital OR of OR Hungary.AND_OPERATOR mode terms are considered to be in conjuction: the
+ * In AND_OPERATOR mode terms are considered to be in conjunction: the
* above mentioned query is parsed as capital AND of AND Hungary
*/
public void setDefaultOperator(Operator op) {
@@ -314,48 +319,40 @@
// PhraseQuery, or nothing based on the term count
TokenStream source = analyzer.tokenStream(field, new StringReader(queryText));
- List list = new ArrayList();
- final org.apache.lucene.analysis.Token reusableToken = new org.apache.lucene.analysis.Token();
- org.apache.lucene.analysis.Token nextToken;
+ List* - * @deprecated in favor of {@link ASCIIFoldingFilter} which covers a superset - * of Latin 1. This class will be removed in Lucene 3.0. + * @deprecated If you build a new index, use {@link ASCIIFoldingFilter} + * which covers a superset of Latin 1. + * This class is included for use with existing + * indexes and will be removed in a future release (possibly Lucene 4.0). */ -public class ISOLatin1AccentFilter extends TokenFilter { +public final class ISOLatin1AccentFilter extends TokenFilter { public ISOLatin1AccentFilter(TokenStream input) { super(input); termAtt = addAttribute(TermAttribute.class); @@ -58,18 +60,6 @@ return false; } - /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should - * not be overridden. Delegates to the backwards compatibility layer. */ - public final Token next(final Token reusableToken) throws java.io.IOException { - return super.next(reusableToken); - } - - /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should - * not be overridden. Delegates to the backwards compatibility layer. */ - public final Token next() throws java.io.IOException { - return super.next(); - } - /** * To replace accented characters in a String by unaccented equivalents. */ Index: src/java/org/apache/lucene/analysis/KeywordTokenizer.java =================================================================== --- src/java/org/apache/lucene/analysis/KeywordTokenizer.java (revision 824110) +++ src/java/org/apache/lucene/analysis/KeywordTokenizer.java (working copy) @@ -27,7 +27,7 @@ /** * Emits the entire input as a single token. */ -public class KeywordTokenizer extends Tokenizer { +public final class KeywordTokenizer extends Tokenizer { private static final int DEFAULT_BUFFER_SIZE = 256; @@ -88,18 +88,6 @@ offsetAtt.setOffset(finalOffset, finalOffset); } - /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should - * not be overridden. Delegates to the backwards compatibility layer. */ - public final Token next(final Token reusableToken) throws IOException { - return super.next(reusableToken); - } - - /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should - * not be overridden. Delegates to the backwards compatibility layer. */ - public final Token next() throws IOException { - return super.next(); - } - public void reset(Reader input) throws IOException { super.reset(input); this.done = false; Index: src/java/org/apache/lucene/analysis/SinkTokenizer.java =================================================================== --- src/java/org/apache/lucene/analysis/SinkTokenizer.java (revision 824110) +++ src/java/org/apache/lucene/analysis/SinkTokenizer.java (working copy) @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.analysis; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - - -/** - * A SinkTokenizer can be used to cache Tokens for use in an Analyzer - *
- * WARNING: {@link TeeTokenFilter} and {@link SinkTokenizer} only work with the old TokenStream API. - * If you switch to the new API, you need to use {@link TeeSinkTokenFilter} instead, which offers - * the same functionality. - * @see TeeTokenFilter - * @deprecated Use {@link TeeSinkTokenFilter} instead - * - **/ -public class SinkTokenizer extends Tokenizer { - protected List/*
-SinkTokenizer sink1 = new SinkTokenizer();
-SinkTokenizer sink2 = new SinkTokenizer();
-
-TokenStream source1 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(reader1), sink1), sink2);
-TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(reader2), sink1), sink2);
-
-TokenStream final1 = new LowerCaseFilter(source1);
-TokenStream final2 = source2;
-TokenStream final3 = new EntityDetect(sink1);
-TokenStream final4 = new URLDetect(sink2);
-
-d.add(new Field("f1", final1));
-d.add(new Field("f2", final2));
-d.add(new Field("f3", final3));
-d.add(new Field("f4", final4));
- *
- * In this example, sink1 and sink2 will both get tokens from both
- * reader1 and reader2 after whitespace tokenizer
- * and now we can further wrap any of these in extra analysis, and more "sources" can be inserted if desired.
- * It is important, that tees are consumed before sinks (in the above example, the field names must be
- * less the sink's field names).
- * Note, the EntityDetect and URLDetect TokenStreams are for the example and do not currently exist in Lucene
-
- *
- * See LUCENE-1058.
- *
- * WARNING: {@link TeeTokenFilter} and {@link SinkTokenizer} only work with the old TokenStream API.
- * If you switch to the new API, you need to use {@link TeeSinkTokenFilter} instead, which offers
- * the same functionality.
-
- * @see SinkTokenizer
- * @deprecated Use {@link TeeSinkTokenFilter} instead
- **/
-public class TeeTokenFilter extends TokenFilter {
- SinkTokenizer sink;
-
- public TeeTokenFilter(TokenStream input, SinkTokenizer sink) {
- super(input);
- this.sink = sink;
- }
-
- public Token next(final Token reusableToken) throws IOException {
- assert reusableToken != null;
- Token nextToken = input.next(reusableToken);
- sink.add(nextToken);
- return nextToken;
- }
-
-}
Index: src/java/org/apache/lucene/analysis/Token.java
===================================================================
--- src/java/org/apache/lucene/analysis/Token.java (revision 824110)
+++ src/java/org/apache/lucene/analysis/Token.java (working copy)
@@ -867,9 +867,6 @@
if (payload !=null) {
to.payload = (Payload) payload.clone();
}
- // remove the following optimization in 3.0 when old TokenStream API removed:
- } else if (target instanceof TokenWrapper) {
- ((TokenWrapper) target).delegate = (Token) this.clone();
} else {
initTermBuffer();
((TermAttribute) target).setTermBuffer(termBuffer, 0, termLength);
Index: src/java/org/apache/lucene/analysis/TokenStream.java
===================================================================
--- src/java/org/apache/lucene/analysis/TokenStream.java (revision 824110)
+++ src/java/org/apache/lucene/analysis/TokenStream.java (working copy)
@@ -18,18 +18,10 @@
*/
import java.io.IOException;
-import java.util.IdentityHashMap;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Payload;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeSource;
@@ -78,103 +70,19 @@
* You can find some example code for the new API in the analysis package level
* Javadoc.
*
- * Sometimes it is desirable to capture a current state of a TokenStream
- * , e. g. for buffering purposes (see {@link CachingTokenFilter},
+ * Sometimes it is desirable to capture a current state of a TokenStream,
+ * e.g. for buffering purposes (see {@link CachingTokenFilter},
* {@link TeeSinkTokenFilter}). For this usecase
* {@link AttributeSource#captureState} and {@link AttributeSource#restoreState}
* can be used.
*/
public abstract class TokenStream extends AttributeSource {
- /** @deprecated Remove this when old API is removed! */
- private static final AttributeFactory DEFAULT_TOKEN_WRAPPER_ATTRIBUTE_FACTORY
- = new TokenWrapperAttributeFactory(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
-
- /** @deprecated Remove this when old API is removed! */
- private final TokenWrapper tokenWrapper;
-
- /** @deprecated Remove this when old API is removed! */
- private static boolean onlyUseNewAPI = false;
-
- /** @deprecated Remove this when old API is removed! */
- private final MethodSupport supportedMethods = getSupportedMethods(this.getClass());
-
- /** @deprecated Remove this when old API is removed! */
- private static final class MethodSupport {
- final boolean hasIncrementToken, hasReusableNext, hasNext;
-
- MethodSupport(Class clazz) {
- hasIncrementToken = isMethodOverridden(clazz, "incrementToken");
- hasReusableNext = isMethodOverridden(clazz, "next", Token.class);
- hasNext = isMethodOverridden(clazz, "next");
- }
-
- private static boolean isMethodOverridden(Class clazz, String name, Class... params) {
- try {
- return clazz.getMethod(name, params).getDeclaringClass() != TokenStream.class;
- } catch (NoSuchMethodException e) {
- // should not happen
- throw new RuntimeException(e);
- }
- }
- }
-
- /** @deprecated Remove this when old API is removed! */
- private static final IdentityHashMap,MethodSupport> knownMethodSupport =
- new IdentityHashMap,MethodSupport>();
-
- /** @deprecated Remove this when old API is removed! */
- private static MethodSupport getSupportedMethods(Class clazz) {
- MethodSupport supportedMethods;
- synchronized(knownMethodSupport) {
- supportedMethods = knownMethodSupport.get(clazz);
- if (supportedMethods == null) {
- knownMethodSupport.put(clazz, supportedMethods = new MethodSupport(clazz));
- }
- }
- return supportedMethods;
- }
-
- /** @deprecated Remove this when old API is removed! */
- private static final class TokenWrapperAttributeFactory extends AttributeFactory {
- private final AttributeFactory delegate;
-
- private TokenWrapperAttributeFactory(AttributeFactory delegate) {
- this.delegate = delegate;
- }
-
- public AttributeImpl createAttributeInstance(Class attClass) {
- return attClass.isAssignableFrom(TokenWrapper.class)
- ? new TokenWrapper()
- : delegate.createAttributeInstance(attClass);
- }
-
- // this is needed for TeeSinkTokenStream's check for compatibility of AttributeSource,
- // so two TokenStreams using old API have the same AttributeFactory wrapped by this one.
- public boolean equals(Object other) {
- if (this == other) return true;
- if (other instanceof TokenWrapperAttributeFactory) {
- final TokenWrapperAttributeFactory af = (TokenWrapperAttributeFactory) other;
- return this.delegate.equals(af.delegate);
- }
- return false;
- }
-
- public int hashCode() {
- return delegate.hashCode() ^ 0x0a45ff31;
- }
- }
-
/**
* A TokenStream using the default attribute factory.
*/
protected TokenStream() {
- super(onlyUseNewAPI
- ? AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY
- : TokenStream.DEFAULT_TOKEN_WRAPPER_ATTRIBUTE_FACTORY
- );
- tokenWrapper = initTokenWrapper(null);
- check();
+ super();
}
/**
@@ -182,105 +90,16 @@
*/
protected TokenStream(AttributeSource input) {
super(input);
- tokenWrapper = initTokenWrapper(input);
- check();
}
/**
* A TokenStream using the supplied AttributeFactory for creating new {@link Attribute} instances.
*/
protected TokenStream(AttributeFactory factory) {
- super(onlyUseNewAPI
- ? factory
- : new TokenWrapperAttributeFactory(factory)
- );
- tokenWrapper = initTokenWrapper(null);
- check();
+ super(factory);
}
-
- /** @deprecated Remove this when old API is removed! */
- private TokenWrapper initTokenWrapper(AttributeSource input) {
- if (onlyUseNewAPI) {
- // no wrapper needed
- return null;
- } else {
- // if possible get the wrapper from the filter's input stream
- if (input instanceof TokenStream && ((TokenStream) input).tokenWrapper != null) {
- return ((TokenStream) input).tokenWrapper;
- }
- // check that all attributes are implemented by the same TokenWrapper instance
- final Attribute att = addAttribute(TermAttribute.class);
- if (att instanceof TokenWrapper &&
- addAttribute(TypeAttribute.class) == att &&
- addAttribute(PositionIncrementAttribute.class) == att &&
- addAttribute(FlagsAttribute.class) == att &&
- addAttribute(OffsetAttribute.class) == att &&
- addAttribute(PayloadAttribute.class) == att
- ) {
- return (TokenWrapper) att;
- } else {
- throw new UnsupportedOperationException(
- "If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class "+
- "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been "+
- "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!"
- );
- }
- }
- }
-
- /** @deprecated Remove this when old API is removed! */
- private void check() {
- if (onlyUseNewAPI && !supportedMethods.hasIncrementToken) {
- throw new UnsupportedOperationException(getClass().getName()+" does not implement incrementToken() which is needed for onlyUseNewAPI.");
- }
-
- // a TokenStream subclass must at least implement one of the methods!
- if (!(supportedMethods.hasIncrementToken || supportedMethods.hasNext || supportedMethods.hasReusableNext)) {
- throw new UnsupportedOperationException(getClass().getName()+" does not implement any of incrementToken(), next(Token), next().");
- }
- }
/**
- * For extra performance you can globally enable the new
- * {@link #incrementToken} API using {@link Attribute}s. There will be a
- * small, but in most cases negligible performance increase by enabling this,
- * but it only works if all TokenStreams use the new API and
- * implement {@link #incrementToken}. This setting can only be enabled
- * globally.
- *
- * This setting only affects TokenStreams instantiated after this
- * call. All TokenStreams already created use the other setting.
- *
- * All core {@link Analyzer}s are compatible with this setting, if you have
- * your own TokenStreams that are also compatible, you should enable
- * this.
- *
- * When enabled, tokenization may throw {@link UnsupportedOperationException}
- * s, if the whole tokenizer chain is not compatible eg one of the
- * TokenStreams does not implement the new TokenStream API.
- *
- * The default is false, so there is the fallback to the old API
- * available.
- *
- * @deprecated This setting will no longer be needed in Lucene 3.0 as the old
- * API will be removed.
- */
- public static void setOnlyUseNewAPI(boolean onlyUseNewAPI) {
- TokenStream.onlyUseNewAPI = onlyUseNewAPI;
- }
-
- /**
- * Returns if only the new API is used.
- *
- * @see #setOnlyUseNewAPI
- * @deprecated This setting will no longer be needed in Lucene 3.0 as
- * the old API will be removed.
- */
- public static boolean getOnlyUseNewAPI() {
- return onlyUseNewAPI;
- }
-
- /**
* Consumers (ie {@link IndexWriter}) use this method to advance the stream to
* the next token. Implementing classes must implement this method and update
* the appropriate {@link AttributeImpl}s with the attributes of the next
@@ -293,7 +112,7 @@
*
* This method is called for every token of a document, so an efficient
* implementation is crucial for good performance. To avoid calls to
- * {@link #addAttribute(Class)} and {@link #getAttribute(Class)} or downcasts,
+ * {@link #addAttribute(Class)} and {@link #getAttribute(Class)},
* references to all {@link AttributeImpl}s that this stream uses should be
* retrieved during instantiation.
*
@@ -303,25 +122,8 @@
* {@link #incrementToken()}.
*
* @return false for end of stream; true otherwise
- *
- *
- * Note that this method will be defined abstract in Lucene
- * 3.0.
*/
- public boolean incrementToken() throws IOException {
- assert tokenWrapper != null;
-
- final Token token;
- if (supportedMethods.hasReusableNext) {
- token = next(tokenWrapper.delegate);
- } else {
- assert supportedMethods.hasNext;
- token = next();
- }
- if (token == null) return false;
- tokenWrapper.delegate = token;
- return true;
- }
+ public abstract boolean incrementToken() throws IOException;
/**
* This method is called by the consumer after the last token has been
@@ -341,83 +143,6 @@
}
/**
- * Returns the next token in the stream, or null at EOS. When possible, the
- * input Token should be used as the returned Token (this gives fastest
- * tokenization performance), but this is not required and a new Token may be
- * returned. Callers may re-use a single Token instance for successive calls
- * to this method.
- *
- * This implicitly defines a "contract" between consumers (callers of this
- * method) and producers (implementations of this method that are the source
- * for tokens):
- *
- * - A consumer must fully consume the previously returned {@link Token}
- * before calling this method again.
- * - A producer must call {@link Token#clear()} before setting the fields in
- * it and returning it
- *
- * Also, the producer must make no assumptions about a {@link Token} after it
- * has been returned: the caller may arbitrarily change it. If the producer
- * needs to hold onto the {@link Token} for subsequent calls, it must clone()
- * it before storing it. Note that a {@link TokenFilter} is considered a
- * consumer.
- *
- * @param reusableToken a {@link Token} that may or may not be used to return;
- * this parameter should never be null (the callee is not required to
- * check for null before using it, but it is a good idea to assert that
- * it is not null.)
- * @return next {@link Token} in the stream or null if end-of-stream was hit
- * @deprecated The new {@link #incrementToken()} and {@link AttributeSource}
- * APIs should be used instead.
- */
- public Token next(final Token reusableToken) throws IOException {
- assert reusableToken != null;
-
- if (tokenWrapper == null)
- throw new UnsupportedOperationException("This TokenStream only supports the new Attributes API.");
-
- if (supportedMethods.hasIncrementToken) {
- tokenWrapper.delegate = reusableToken;
- return incrementToken() ? tokenWrapper.delegate : null;
- } else {
- assert supportedMethods.hasNext;
- return next();
- }
- }
-
- /**
- * Returns the next {@link Token} in the stream, or null at EOS.
- *
- * @deprecated The returned Token is a "full private copy" (not re-used across
- * calls to {@link #next()}) but will be slower than calling
- * {@link #next(Token)} or using the new {@link #incrementToken()}
- * method with the new {@link AttributeSource} API.
- */
- public Token next() throws IOException {
- if (tokenWrapper == null)
- throw new UnsupportedOperationException("This TokenStream only supports the new Attributes API.");
-
- final Token nextToken;
- if (supportedMethods.hasIncrementToken) {
- final Token savedDelegate = tokenWrapper.delegate;
- tokenWrapper.delegate = new Token();
- nextToken = incrementToken() ? tokenWrapper.delegate : null;
- tokenWrapper.delegate = savedDelegate;
- } else {
- assert supportedMethods.hasReusableNext;
- nextToken = next(new Token());
- }
-
- if (nextToken != null) {
- Payload p = nextToken.getPayload();
- if (p != null) {
- nextToken.setPayload((Payload) p.clone());
- }
- }
- return nextToken;
- }
-
- /**
* Resets this stream to the beginning. This is an optional operation, so
* subclasses may or may not implement this method. {@link #reset()} is not needed for
* the standard indexing process. However, if the tokens of a
Index: src/java/org/apache/lucene/analysis/TokenWrapper.java
===================================================================
--- src/java/org/apache/lucene/analysis/TokenWrapper.java (revision 824110)
+++ src/java/org/apache/lucene/analysis/TokenWrapper.java (working copy)
@@ -1,166 +0,0 @@
-package org.apache.lucene.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.index.Payload;
-import org.apache.lucene.util.AttributeImpl;
-
-/**
- * This class wraps a Token and supplies a single attribute instance
- * where the delegate token can be replaced.
- * @deprecated Will be removed, when old TokenStream API is removed.
- */
-final class TokenWrapper extends AttributeImpl
- implements Cloneable, TermAttribute, TypeAttribute, PositionIncrementAttribute,
- FlagsAttribute, OffsetAttribute, PayloadAttribute {
-
- Token delegate;
-
- TokenWrapper() {
- this(new Token());
- }
-
- TokenWrapper(Token delegate) {
- this.delegate = delegate;
- }
-
- // TermAttribute:
-
- public String term() {
- return delegate.term();
- }
-
- public void setTermBuffer(char[] buffer, int offset, int length) {
- delegate.setTermBuffer(buffer, offset, length);
- }
-
- public void setTermBuffer(String buffer) {
- delegate.setTermBuffer(buffer);
- }
-
- public void setTermBuffer(String buffer, int offset, int length) {
- delegate.setTermBuffer(buffer, offset, length);
- }
-
- public char[] termBuffer() {
- return delegate.termBuffer();
- }
-
- public char[] resizeTermBuffer(int newSize) {
- return delegate.resizeTermBuffer(newSize);
- }
-
- public int termLength() {
- return delegate.termLength();
- }
-
- public void setTermLength(int length) {
- delegate.setTermLength(length);
- }
-
- // TypeAttribute:
-
- public String type() {
- return delegate.type();
- }
-
- public void setType(String type) {
- delegate.setType(type);
- }
-
- public void setPositionIncrement(int positionIncrement) {
- delegate.setPositionIncrement(positionIncrement);
- }
-
- public int getPositionIncrement() {
- return delegate.getPositionIncrement();
- }
-
- // FlagsAttribute
-
- public int getFlags() {
- return delegate.getFlags();
- }
-
- public void setFlags(int flags) {
- delegate.setFlags(flags);
- }
-
- // OffsetAttribute
-
- public int startOffset() {
- return delegate.startOffset();
- }
-
- public void setOffset(int startOffset, int endOffset) {
- delegate.setOffset(startOffset, endOffset);
- }
-
- public int endOffset() {
- return delegate.endOffset();
- }
-
- // PayloadAttribute
-
- public Payload getPayload() {
- return delegate.getPayload();
- }
-
- public void setPayload(Payload payload) {
- delegate.setPayload(payload);
- }
-
- // AttributeImpl
-
- public void clear() {
- delegate.clear();
- }
-
- public String toString() {
- return delegate.toString();
- }
-
- public int hashCode() {
- return delegate.hashCode();
- }
-
- public boolean equals(Object other) {
- if (other instanceof TokenWrapper) {
- return ((TokenWrapper) other).delegate.equals(this.delegate);
- }
- return false;
- }
-
- public Object clone() {
- return new TokenWrapper((Token) delegate.clone());
- }
-
- public void copyTo(AttributeImpl target) {
- if (target instanceof TokenWrapper) {
- ((TokenWrapper) target).delegate = (Token) this.delegate.clone();
- } else {
- this.delegate.copyTo(target);
- }
- }
-}
Property changes on: src\test\org\apache\lucene\analysis\TestISOLatin1AccentFilter.java
___________________________________________________________________
Added: svn:mergeinfo
Merged /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:r748824
Index: src/test/org/apache/lucene/analysis/TestToken.java
===================================================================
--- src/test/org/apache/lucene/analysis/TestToken.java (revision 824110)
+++ src/test/org/apache/lucene/analysis/TestToken.java (working copy)
@@ -18,9 +18,13 @@
*/
import org.apache.lucene.index.Payload;
-import org.apache.lucene.analysis.tokenattributes.TestSimpleAttributeImpls;
+import org.apache.lucene.analysis.tokenattributes.*;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import java.io.StringReader;
+
public class TestToken extends LuceneTestCase {
public TestToken(String name) {
@@ -219,4 +223,33 @@
assertEquals(pl, copy.getPayload());
assertNotSame(pl, copy.getPayload());
}
+
+ public interface SenselessAttribute extends Attribute {}
+
+ public static final class SenselessAttributeImpl extends AttributeImpl implements SenselessAttribute {
+ public void copyTo(AttributeImpl target) {}
+ public void clear() {}
+ public boolean equals(Object o) { return (o instanceof SenselessAttributeImpl); }
+ public int hashCode() { return 0; }
+ }
+
+ public void testTokenAttributeFactory() throws Exception {
+ TokenStream ts = new WhitespaceTokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, new StringReader("foo bar"));
+
+ assertTrue("TypeAttribute is not implemented by SenselessAttributeImpl",
+ ts.addAttribute(SenselessAttribute.class) instanceof SenselessAttributeImpl);
+
+ assertTrue("TermAttribute is not implemented by Token",
+ ts.addAttribute(TermAttribute.class) instanceof Token);
+ assertTrue("OffsetAttribute is not implemented by Token",
+ ts.addAttribute(OffsetAttribute.class) instanceof Token);
+ assertTrue("FlagsAttribute is not implemented by Token",
+ ts.addAttribute(FlagsAttribute.class) instanceof Token);
+ assertTrue("PayloadAttribute is not implemented by Token",
+ ts.addAttribute(PayloadAttribute.class) instanceof Token);
+ assertTrue("PositionIncrementAttribute is not implemented by Token",
+ ts.addAttribute(PositionIncrementAttribute.class) instanceof Token);
+ assertTrue("TypeAttribute is not implemented by Token",
+ ts.addAttribute(TypeAttribute.class) instanceof Token);
+ }
}
Index: src/test/org/apache/lucene/util/TestAttributeSource.java
===================================================================
--- src/test/org/apache/lucene/util/TestAttributeSource.java (revision 824110)
+++ src/test/org/apache/lucene/util/TestAttributeSource.java (working copy)
@@ -125,4 +125,20 @@
assertEquals("Token should only printed once", "("+tok.toString()+")", src.toString());
}
+ public void testDefaultAttributeFactory() throws Exception {
+ AttributeSource src = new AttributeSource();
+
+ assertTrue("TermAttribute is not implemented by TermAttributeImpl",
+ src.addAttribute(TermAttribute.class) instanceof TermAttributeImpl);
+ assertTrue("OffsetAttribute is not implemented by OffsetAttributeImpl",
+ src.addAttribute(OffsetAttribute.class) instanceof OffsetAttributeImpl);
+ assertTrue("FlagsAttribute is not implemented by FlagsAttributeImpl",
+ src.addAttribute(FlagsAttribute.class) instanceof FlagsAttributeImpl);
+ assertTrue("PayloadAttribute is not implemented by PayloadAttributeImpl",
+ src.addAttribute(PayloadAttribute.class) instanceof PayloadAttributeImpl);
+ assertTrue("PositionIncrementAttribute is not implemented by PositionIncrementAttributeImpl",
+ src.addAttribute(PositionIncrementAttribute.class) instanceof PositionIncrementAttributeImpl);
+ assertTrue("TypeAttribute is not implemented by TypeAttributeImpl",
+ src.addAttribute(TypeAttribute.class) instanceof TypeAttributeImpl);
+ }
}