",
"<_NUM_CHAR>",
Index: contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java
===================================================================
--- contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java (revision 824063)
+++ contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java (working copy)
@@ -10,6 +10,7 @@
import java.util.Vector;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.*;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
@@ -23,10 +24,15 @@
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.Parameter;
+import org.apache.lucene.util.AttributeSource;
+/** Token Manager. */
public class PrecedenceQueryParserTokenManager implements PrecedenceQueryParserConstants
{
+
+ /** Debug output. */
public java.io.PrintStream debugStream = System.out;
+ /** Set debug output. */
public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
private final int jjStopStringLiteralDfa_3(int pos, long active0)
{
@@ -40,22 +46,14 @@
{
return jjMoveNfa_3(jjStopStringLiteralDfa_3(pos, active0), pos + 1);
}
-private final int jjStopAtPos(int pos, int kind)
+private int jjStopAtPos(int pos, int kind)
{
jjmatchedKind = kind;
jjmatchedPos = pos;
return pos + 1;
}
-private final int jjStartNfaWithStates_3(int pos, int kind, int state)
+private int jjMoveStringLiteralDfa0_3()
{
- jjmatchedKind = kind;
- jjmatchedPos = pos;
- try { curChar = input_stream.readChar(); }
- catch(java.io.IOException e) { return pos + 1; }
- return jjMoveNfa_3(state, pos + 1);
-}
-private final int jjMoveStringLiteralDfa0_3()
-{
switch(curChar)
{
case 40:
@@ -78,50 +76,19 @@
return jjMoveNfa_3(0, 0);
}
}
-private final void jjCheckNAdd(int state)
-{
- if (jjrounds[state] != jjround)
- {
- jjstateSet[jjnewStateCnt++] = state;
- jjrounds[state] = jjround;
- }
-}
-private final void jjAddStates(int start, int end)
-{
- do {
- jjstateSet[jjnewStateCnt++] = jjnextStates[start];
- } while (start++ != end);
-}
-private final void jjCheckNAddTwoStates(int state1, int state2)
-{
- jjCheckNAdd(state1);
- jjCheckNAdd(state2);
-}
-private final void jjCheckNAddStates(int start, int end)
-{
- do {
- jjCheckNAdd(jjnextStates[start]);
- } while (start++ != end);
-}
-private final void jjCheckNAddStates(int start)
-{
- jjCheckNAdd(jjnextStates[start]);
- jjCheckNAdd(jjnextStates[start + 1]);
-}
static final long[] jjbitVec0 = {
0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
};
static final long[] jjbitVec2 = {
0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
};
-private final int jjMoveNfa_3(int startState, int curPos)
+private int jjMoveNfa_3(int startState, int curPos)
{
- int[] nextStates;
int startsAt = 0;
jjnewStateCnt = 33;
int i = 1;
jjstateSet[0] = startState;
- int j, kind = 0x7fffffff;
+ int kind = 0x7fffffff;
for (;;)
{
if (++jjround == 0x7fffffff)
@@ -129,7 +96,7 @@
if (curChar < 64)
{
long l = 1L << curChar;
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -251,7 +218,7 @@
else if (curChar < 128)
{
long l = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -399,7 +366,7 @@
long l1 = 1L << (hiByte & 077);
int i2 = (curChar & 0xff) >> 6;
long l2 = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -468,16 +435,8 @@
{
return jjMoveNfa_1(jjStopStringLiteralDfa_1(pos, active0), pos + 1);
}
-private final int jjStartNfaWithStates_1(int pos, int kind, int state)
+private int jjMoveStringLiteralDfa0_1()
{
- jjmatchedKind = kind;
- jjmatchedPos = pos;
- try { curChar = input_stream.readChar(); }
- catch(java.io.IOException e) { return pos + 1; }
- return jjMoveNfa_1(state, pos + 1);
-}
-private final int jjMoveStringLiteralDfa0_1()
-{
switch(curChar)
{
case 84:
@@ -488,7 +447,7 @@
return jjMoveNfa_1(0, 0);
}
}
-private final int jjMoveStringLiteralDfa1_1(long active0)
+private int jjMoveStringLiteralDfa1_1(long active0)
{
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) {
@@ -506,14 +465,21 @@
}
return jjStartNfa_1(0, active0);
}
-private final int jjMoveNfa_1(int startState, int curPos)
+private int jjStartNfaWithStates_1(int pos, int kind, int state)
{
- int[] nextStates;
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) { return pos + 1; }
+ return jjMoveNfa_1(state, pos + 1);
+}
+private int jjMoveNfa_1(int startState, int curPos)
+{
int startsAt = 0;
jjnewStateCnt = 5;
int i = 1;
jjstateSet[0] = startState;
- int j, kind = 0x7fffffff;
+ int kind = 0x7fffffff;
for (;;)
{
if (++jjround == 0x7fffffff)
@@ -521,7 +487,7 @@
if (curChar < 64)
{
long l = 1L << curChar;
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -566,7 +532,7 @@
else if (curChar < 128)
{
long l = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -592,7 +558,7 @@
long l1 = 1L << (hiByte & 077);
int i2 = (curChar & 0xff) >> 6;
long l2 = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -625,18 +591,17 @@
catch(java.io.IOException e) { return curPos; }
}
}
-private final int jjMoveStringLiteralDfa0_0()
+private int jjMoveStringLiteralDfa0_0()
{
return jjMoveNfa_0(0, 0);
}
-private final int jjMoveNfa_0(int startState, int curPos)
+private int jjMoveNfa_0(int startState, int curPos)
{
- int[] nextStates;
int startsAt = 0;
jjnewStateCnt = 3;
int i = 1;
jjstateSet[0] = startState;
- int j, kind = 0x7fffffff;
+ int kind = 0x7fffffff;
for (;;)
{
if (++jjround == 0x7fffffff)
@@ -644,7 +609,7 @@
if (curChar < 64)
{
long l = 1L << curChar;
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -673,7 +638,7 @@
else if (curChar < 128)
{
long l = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -688,7 +653,7 @@
long l1 = 1L << (hiByte & 077);
int i2 = (curChar & 0xff) >> 6;
long l2 = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -728,16 +693,8 @@
{
return jjMoveNfa_2(jjStopStringLiteralDfa_2(pos, active0), pos + 1);
}
-private final int jjStartNfaWithStates_2(int pos, int kind, int state)
+private int jjMoveStringLiteralDfa0_2()
{
- jjmatchedKind = kind;
- jjmatchedPos = pos;
- try { curChar = input_stream.readChar(); }
- catch(java.io.IOException e) { return pos + 1; }
- return jjMoveNfa_2(state, pos + 1);
-}
-private final int jjMoveStringLiteralDfa0_2()
-{
switch(curChar)
{
case 84:
@@ -748,7 +705,7 @@
return jjMoveNfa_2(0, 0);
}
}
-private final int jjMoveStringLiteralDfa1_2(long active0)
+private int jjMoveStringLiteralDfa1_2(long active0)
{
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) {
@@ -766,14 +723,21 @@
}
return jjStartNfa_2(0, active0);
}
-private final int jjMoveNfa_2(int startState, int curPos)
+private int jjStartNfaWithStates_2(int pos, int kind, int state)
{
- int[] nextStates;
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ try { curChar = input_stream.readChar(); }
+ catch(java.io.IOException e) { return pos + 1; }
+ return jjMoveNfa_2(state, pos + 1);
+}
+private int jjMoveNfa_2(int startState, int curPos)
+{
int startsAt = 0;
jjnewStateCnt = 5;
int i = 1;
jjstateSet[0] = startState;
- int j, kind = 0x7fffffff;
+ int kind = 0x7fffffff;
for (;;)
{
if (++jjround == 0x7fffffff)
@@ -781,7 +745,7 @@
if (curChar < 64)
{
long l = 1L << curChar;
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -826,7 +790,7 @@
else if (curChar < 128)
{
long l = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -852,7 +816,7 @@
long l1 = 1L << (hiByte & 077);
int i2 = (curChar & 0xff) >> 6;
long l2 = 1L << (curChar & 077);
- MatchLoop: do
+ do
{
switch(jjstateSet[--i])
{
@@ -895,22 +859,28 @@
{
case 0:
return ((jjbitVec2[i2] & l2) != 0L);
- default :
+ default :
if ((jjbitVec0[i1] & l1) != 0L)
return true;
return false;
}
}
+
+/** Token literal values. */
public static final String[] jjstrLiteralImages = {
"", null, null, null, null, null, null, null, null, null, "\53", "\55", "\50",
"\51", "\72", "\136", null, null, null, null, null, "\133", "\173", null, "\124\117",
"\135", null, null, "\124\117", "\175", null, null, };
+
+/** Lexer state names. */
public static final String[] lexStateNames = {
- "Boost",
- "RangeEx",
- "RangeIn",
- "DEFAULT",
+ "Boost",
+ "RangeEx",
+ "RangeIn",
+ "DEFAULT",
};
+
+/** Lex State array. */
public static final int[] jjnewLexState = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, 2, 1, 3, -1,
3, -1, -1, -1, 3, -1, -1,
@@ -925,15 +895,18 @@
private final int[] jjrounds = new int[33];
private final int[] jjstateSet = new int[66];
protected char curChar;
-public PrecedenceQueryParserTokenManager(CharStream stream)
-{
+/** Constructor. */
+public PrecedenceQueryParserTokenManager(CharStream stream){
input_stream = stream;
}
-public PrecedenceQueryParserTokenManager(CharStream stream, int lexState)
-{
+
+/** Constructor. */
+public PrecedenceQueryParserTokenManager(CharStream stream, int lexState){
this(stream);
SwitchTo(lexState);
}
+
+/** Reinitialise parser. */
public void ReInit(CharStream stream)
{
jjmatchedPos = jjnewStateCnt = 0;
@@ -941,18 +914,22 @@
input_stream = stream;
ReInitRounds();
}
-private final void ReInitRounds()
+private void ReInitRounds()
{
int i;
jjround = 0x80000001;
for (i = 33; i-- > 0;)
jjrounds[i] = 0x80000000;
}
+
+/** Reinitialise parser. */
public void ReInit(CharStream stream, int lexState)
{
ReInit(stream);
SwitchTo(lexState);
}
+
+/** Switch to specified lex state. */
public void SwitchTo(int lexState)
{
if (lexState >= 4 || lexState < 0)
@@ -963,14 +940,25 @@
protected Token jjFillToken()
{
- Token t = Token.newToken(jjmatchedKind);
- t.kind = jjmatchedKind;
+ final Token t;
+ final String curTokenImage;
+ final int beginLine;
+ final int endLine;
+ final int beginColumn;
+ final int endColumn;
String im = jjstrLiteralImages[jjmatchedKind];
- t.image = (im == null) ? input_stream.GetImage() : im;
- t.beginLine = input_stream.getBeginLine();
- t.beginColumn = input_stream.getBeginColumn();
- t.endLine = input_stream.getEndLine();
- t.endColumn = input_stream.getEndColumn();
+ curTokenImage = (im == null) ? input_stream.GetImage() : im;
+ beginLine = input_stream.getBeginLine();
+ beginColumn = input_stream.getBeginColumn();
+ endLine = input_stream.getEndLine();
+ endColumn = input_stream.getEndColumn();
+ t = Token.newToken(jjmatchedKind, curTokenImage);
+
+ t.beginLine = beginLine;
+ t.endLine = endLine;
+ t.beginColumn = beginColumn;
+ t.endColumn = endColumn;
+
return t;
}
@@ -981,22 +969,21 @@
int jjmatchedPos;
int jjmatchedKind;
+/** Get the next Token. */
public Token getNextToken()
{
- int kind;
- Token specialToken = null;
Token matchedToken;
int curPos = 0;
EOFLoop :
for (;;)
- {
- try
- {
+ {
+ try
+ {
curChar = input_stream.BeginToken();
- }
+ }
catch(java.io.IOException e)
- {
+ {
jjmatchedKind = 0;
matchedToken = jjFillToken();
return matchedToken;
@@ -1066,4 +1053,31 @@
}
}
+private void jjCheckNAdd(int state)
+{
+ if (jjrounds[state] != jjround)
+ {
+ jjstateSet[jjnewStateCnt++] = state;
+ jjrounds[state] = jjround;
+ }
}
+private void jjAddStates(int start, int end)
+{
+ do {
+ jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+ } while (start++ != end);
+}
+private void jjCheckNAddTwoStates(int state1, int state2)
+{
+ jjCheckNAdd(state1);
+ jjCheckNAdd(state2);
+}
+
+private void jjCheckNAddStates(int start, int end)
+{
+ do {
+ jjCheckNAdd(jjnextStates[start]);
+ } while (start++ != end);
+}
+
+}
Index: contrib/misc/src/java/org/apache/lucene/queryParser/precedence/Token.java
===================================================================
--- contrib/misc/src/java/org/apache/lucene/queryParser/precedence/Token.java (revision 824063)
+++ contrib/misc/src/java/org/apache/lucene/queryParser/precedence/Token.java (working copy)
@@ -1,4 +1,5 @@
-/* Generated By:JavaCC: Do not edit this line. Token.java Version 3.0 */
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 4.1 */
+/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null */
package org.apache.lucene.queryParser.precedence;
/**
@@ -14,12 +15,14 @@
*/
public int kind;
- /**
- * beginLine and beginColumn describe the position of the first character
- * of this token; endLine and endColumn describe the position of the
- * last character of this token.
- */
- public int beginLine, beginColumn, endLine, endColumn;
+ /** The line number of the first character of this Token. */
+ public int beginLine;
+ /** The column number of the first character of this Token. */
+ public int beginColumn;
+ /** The line number of the last character of this Token. */
+ public int endLine;
+ /** The column number of the last character of this Token. */
+ public int endColumn;
/**
* The string image of the token.
@@ -51,6 +54,40 @@
public Token specialToken;
/**
+ * An optional attribute value of the Token.
+ * Tokens which are not used as syntactic sugar will often contain
+ * meaningful values that will be used later on by the compiler or
+ * interpreter. This attribute value is often different from the image.
+ * Any subclass of Token that actually wants to return a non-null value can
+ * override this method as appropriate.
+ */
+ public Object getValue() {
+ return null;
+ }
+
+ /**
+ * No-argument constructor
+ */
+ public Token() {}
+
+ /**
+ * Constructs a new token for the specified Image.
+ */
+ public Token(int kind)
+ {
+ this(kind, null);
+ }
+
+ /**
+ * Constructs a new token for the specified Image and Kind.
+ */
+ public Token(int kind, String image)
+ {
+ this.kind = kind;
+ this.image = image;
+ }
+
+ /**
* Returns the image.
*/
public String toString()
@@ -63,19 +100,25 @@
* can create and return subclass objects based on the value of ofKind.
* Simply add the cases to the switch for all those special cases.
* For example, if you have a subclass of Token called IDToken that
- * you want to create if ofKind is ID, simlpy add something like :
+ * you want to create if ofKind is ID, simply add something like :
*
- * case MyParserConstants.ID : return new IDToken();
+ * case MyParserConstants.ID : return new IDToken(ofKind, image);
*
* to the following switch statement. Then you can cast matchedToken
- * variable to the appropriate type and use it in your lexical actions.
+ * variable to the appropriate type and use sit in your lexical actions.
*/
- public static final Token newToken(int ofKind)
+ public static Token newToken(int ofKind, String image)
{
switch(ofKind)
{
- default : return new Token();
+ default : return new Token(ofKind, image);
}
}
+ public static Token newToken(int ofKind)
+ {
+ return newToken(ofKind, null);
+ }
+
}
+/* JavaCC - OriginalChecksum=0dc5808f2ab8aac8775ea9175fa2cb51 (do not edit this line) */
Index: contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java
===================================================================
--- contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java (revision 824063)
+++ contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java (working copy)
@@ -1,19 +1,22 @@
-/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 4.1 */
+/* JavaCCOptions: */
package org.apache.lucene.queryParser.precedence;
+/** Token Manager Error. */
public class TokenMgrError extends Error
{
+
/*
* Ordinals for various reasons why an Error of this type can be thrown.
*/
/**
- * Lexical error occured.
+ * Lexical error occurred.
*/
static final int LEXICAL_ERROR = 0;
/**
- * An attempt wass made to create a second instance of a static token manager.
+ * An attempt was made to create a second instance of a static token manager.
*/
static final int STATIC_LEXER_ERROR = 1;
@@ -34,7 +37,7 @@
int errorCode;
/**
- * Replaces unprintable characters by their espaced (or unicode escaped)
+ * Replaces unprintable characters by their escaped (or unicode escaped)
* equivalents in the given string
*/
protected static final String addEscapes(String str) {
@@ -85,12 +88,12 @@
/**
* Returns a detailed message for the Error when it is thrown by the
* token manager to indicate a lexical error.
- * Parameters :
- * EOFSeen : indicates if EOF caused the lexicl error
- * curLexState : lexical state in which this error occured
- * errorLine : line number when the error occured
- * errorColumn : column number when the error occured
- * errorAfter : prefix that was seen before this error occured
+ * Parameters :
+ * EOFSeen : indicates if EOF caused the lexical error
+ * curLexState : lexical state in which this error occurred
+ * errorLine : line number when the error occurred
+ * errorColumn : column number when the error occurred
+ * errorAfter : prefix that was seen before this error occurred
* curchar : the offending character
* Note: You can customize the lexical error message by modifying this method.
*/
@@ -105,7 +108,7 @@
/**
* You can also modify the body of this method to customize your error messages.
* For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
- * of end-users concern, so you can return something like :
+ * of end-users concern, so you can return something like :
*
* "Internal Error : Please file a bug report .... "
*
@@ -119,15 +122,19 @@
* Constructors of various flavors follow.
*/
+ /** No arg constructor. */
public TokenMgrError() {
}
+ /** Constructor with message and reason. */
public TokenMgrError(String message, int reason) {
super(message);
errorCode = reason;
}
+ /** Full Constructor. */
public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
}
}
+/* JavaCC - OriginalChecksum=43f834e732189cec173ab002c94ae97f (do not edit this line) */
Index: contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java
===================================================================
--- contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java (revision 824063)
+++ contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java (working copy)
@@ -22,7 +22,7 @@
import junit.framework.TestCase;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.ISOLatin1AccentFilter;
+import org.apache.lucene.analysis.ASCIIFoldingFilter;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardFilter;
@@ -111,7 +111,7 @@
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new StandardTokenizer(reader);
result = new StandardFilter(result);
- result = new ISOLatin1AccentFilter(result);
+ result = new ASCIIFoldingFilter(result);
result = new LowerCaseFilter(result);
return result;
}
Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java
===================================================================
--- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java (revision 824063)
+++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java (working copy)
@@ -181,48 +181,6 @@
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.lucene.analysis.TokenStream#next(org.apache.lucene.analysis
- * .Token)
- */
- @Override
- public Token next(Token reusableToken) throws IOException {
-
- if (multiToken > 0) {
- reusableToken.setTermBuffer("multi" + (multiToken + 1));
- reusableToken.setStartOffset(prevStartOffset);
- reusableToken.setEndOffset(prevEndOffset);
- reusableToken.setType(prevType);
- reusableToken.setPositionIncrement(0);
- multiToken--;
- return reusableToken;
- } else {
- boolean next = (reusableToken = input.next(token)) != null;
- if (next == false) {
- return null;
- }
- prevType = reusableToken.type();
- prevStartOffset = reusableToken.startOffset();
- prevEndOffset = reusableToken.endOffset();
- String text = reusableToken.term();
- if (text.equals("triplemulti")) {
- multiToken = 2;
- return reusableToken;
- } else if (text.equals("multi")) {
- multiToken = 1;
- return reusableToken;
- } else {
- return reusableToken;
- }
- }
-
- }
-
- private Token token = new Token();
-
public final boolean incrementToken() throws java.io.IOException {
if (multiToken > 0) {
termAtt.setTermBuffer("multi" + (multiToken + 1));
@@ -282,30 +240,6 @@
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
}
- private Token token = new Token();
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lucene.analysis.TokenStream#next()
- */
- @Override
- public Token next(Token reusableToken) throws IOException {
- while (null != (reusableToken = input.next(token))) {
- String term = reusableToken.term();
- if (term.equals("the")) {
- // stopword, do nothing
- } else if (term.equals("quick")) {
- reusableToken.setPositionIncrement(2);
- return reusableToken;
- } else {
- reusableToken.setPositionIncrement(1);
- return reusableToken;
- }
- }
- return null;
- }
-
public final boolean incrementToken() throws java.io.IOException {
while (input.incrementToken()) {
if (termAtt.term().equals("the")) {
Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java
===================================================================
--- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java (revision 824063)
+++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java (working copy)
@@ -175,48 +175,6 @@
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.lucene.analysis.TokenStream#next(org.apache.lucene.analysis
- * .Token)
- */
- @Override
- public Token next(Token reusableToken) throws IOException {
-
- if (multiToken > 0) {
- reusableToken.setTermBuffer("multi" + (multiToken + 1));
- reusableToken.setStartOffset(prevStartOffset);
- reusableToken.setEndOffset(prevEndOffset);
- reusableToken.setType(prevType);
- reusableToken.setPositionIncrement(0);
- multiToken--;
- return reusableToken;
- } else {
- boolean next = (reusableToken = input.next(token)) != null;
- if (next == false) {
- return null;
- }
- prevType = reusableToken.type();
- prevStartOffset = reusableToken.startOffset();
- prevEndOffset = reusableToken.endOffset();
- String text = reusableToken.term();
- if (text.equals("triplemulti")) {
- multiToken = 2;
- return reusableToken;
- } else if (text.equals("multi")) {
- multiToken = 1;
- return reusableToken;
- } else {
- return reusableToken;
- }
- }
-
- }
-
- private Token token = new Token();
-
public final boolean incrementToken() throws java.io.IOException {
if (multiToken > 0) {
termAtt.setTermBuffer("multi" + (multiToken + 1));
@@ -276,30 +234,6 @@
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
}
- private Token token = new Token();
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lucene.analysis.TokenStream#next()
- */
- @Override
- public Token next(Token reusableToken) throws IOException {
- while (null != (reusableToken = input.next(token))) {
- String term = reusableToken.term();
- if (term.equals("the")) {
- // stopword, do nothing
- } else if (term.equals("quick")) {
- reusableToken.setPositionIncrement(2);
- return reusableToken;
- } else {
- reusableToken.setPositionIncrement(1);
- return reusableToken;
- }
- }
- return null;
- }
-
public final boolean incrementToken() throws java.io.IOException {
while (input.incrementToken()) {
if (termAtt.term().equals("the")) {
Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java
===================================================================
--- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java (revision 824063)
+++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java (working copy)
@@ -359,8 +359,8 @@
}
private static class EmptyTokenStream extends TokenStream {
- public Token next(final Token reusableToken) {
- return null;
+ public boolean incrementToken() {
+ return false;
}
}
}
Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java
===================================================================
--- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java (revision 824063)
+++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java (working copy)
@@ -357,8 +357,8 @@
}
private static class EmptyTokenStream extends TokenStream {
- public Token next(final Token reusableToken) {
- return null;
+ public boolean incrementToken() {
+ return false;
}
}
}
Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
===================================================================
--- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (revision 824063)
+++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (working copy)
@@ -113,33 +113,6 @@
boolean inPhrase = false;
int savedStart = 0, savedEnd = 0;
- public Token next(Token reusableToken) throws IOException {
- Token token = reusableToken;
-
- if (inPhrase) {
- inPhrase = false;
- token.setTermBuffer("phrase2");
- token.setStartOffset(savedStart);
- token.setEndOffset(savedEnd);
- return reusableToken;
- } else
- while ((token = this.input.next(reusableToken)) != null) {
- if (token.term().equals("phrase")) {
- inPhrase = true;
- savedStart = token.startOffset();
- savedEnd = token.endOffset();
- token.setTermBuffer("phrase1");
- token.setStartOffset(savedStart);
- token.setEndOffset(savedEnd);
- return token;
- } else if (!token.term().equals("stop"))
- return token;
- }
-
- return null;
-
- }
-
public boolean incrementToken() throws IOException {
if (inPhrase) {
inPhrase = false;
@@ -1185,12 +1158,12 @@
private class CannedTokenStream extends TokenStream {
private int upto = 0;
+ final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
+ final TermAttribute term = addAttribute(TermAttribute.class);
public boolean incrementToken() {
if (upto == 4) {
return false;
}
- PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
- TermAttribute term = addAttribute(TermAttribute.class);
if (upto == 0) {
posIncr.setPositionIncrement(1);
term.setTermBuffer("a");
Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
===================================================================
--- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (revision 824063)
+++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (working copy)
@@ -108,33 +108,6 @@
boolean inPhrase = false;
int savedStart = 0, savedEnd = 0;
- public Token next(Token reusableToken) throws IOException {
- Token token = reusableToken;
-
- if (inPhrase) {
- inPhrase = false;
- token.setTermBuffer("phrase2");
- token.setStartOffset(savedStart);
- token.setEndOffset(savedEnd);
- return reusableToken;
- } else
- while ((token = this.input.next(reusableToken)) != null) {
- if (token.term().equals("phrase")) {
- inPhrase = true;
- savedStart = token.startOffset();
- savedEnd = token.endOffset();
- token.setTermBuffer("phrase1");
- token.setStartOffset(savedStart);
- token.setEndOffset(savedEnd);
- return token;
- } else if (!token.term().equals("stop"))
- return token;
- }
-
- return null;
-
- }
-
public boolean incrementToken() throws IOException {
if (inPhrase) {
inPhrase = false;
Index: contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
===================================================================
--- contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java (revision 824063)
+++ contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java (working copy)
@@ -30,7 +30,7 @@
*
* Available stemmers are listed in {@link org.tartarus.snowball.ext}.
*/
-public class SnowballFilter extends TokenFilter {
+public final class SnowballFilter extends TokenFilter {
private SnowballProgram stemmer;
@@ -78,16 +78,4 @@
return false;
}
}
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next(final Token reusableToken) throws java.io.IOException {
- return super.next(reusableToken);
- }
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next() throws java.io.IOException {
- return super.next();
- }
}
Index: contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
===================================================================
--- contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java (revision 824063)
+++ contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java (working copy)
@@ -39,7 +39,7 @@
* EXPERIMENTAL !!!!!!!!!
* NOTE: This Tokenizer is considered experimental and the grammar is subject to change in the trunk and in follow up releases.
*/
-public class WikipediaTokenizer extends Tokenizer {
+public final class WikipediaTokenizer extends Tokenizer {
public static final String INTERNAL_LINK = "il";
public static final String EXTERNAL_LINK = "el";
//The URL part of the link, i.e. the first token
@@ -187,18 +187,6 @@
this.termAtt = addAttribute(TermAttribute.class);
this.flagsAtt = addAttribute(FlagsAttribute.class);
}
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next(final Token reusableToken) throws java.io.IOException {
- return super.next(reusableToken);
- }
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next() throws java.io.IOException {
- return super.next();
- }
/*
* (non-Javadoc)
Index: src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
===================================================================
--- src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (working copy)
@@ -50,11 +50,6 @@
*
*
* See: http://en.wikipedia.org/wiki/Latin_characters_in_Unicode
- *
- * The set of character conversions supported by this class is a superset of
- * those supported by Lucene's {@link ISOLatin1AccentFilter} which strips
- * accents from Latin1 characters. For example, 'à' will be replaced by
- * 'a'.
*/
public final class ASCIIFoldingFilter extends TokenFilter {
public ASCIIFoldingFilter(TokenStream input)
Index: src/java/org/apache/lucene/analysis/CachingTokenFilter.java
===================================================================
--- src/java/org/apache/lucene/analysis/CachingTokenFilter.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/CachingTokenFilter.java (working copy)
@@ -33,7 +33,7 @@
* {@link TokenStream#reset()}, which repositions the
* stream to the first Token.
*/
-public class CachingTokenFilter extends TokenFilter {
+public final class CachingTokenFilter extends TokenFilter {
private List cache = null;
private Iterator iterator = null;
private AttributeSource.State finalState;
@@ -41,18 +41,6 @@
public CachingTokenFilter(TokenStream input) {
super(input);
}
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next(final Token reusableToken) throws IOException {
- return super.next(reusableToken);
- }
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next() throws IOException {
- return super.next();
- }
public final boolean incrementToken() throws IOException {
if (cache == null) {
Index: src/java/org/apache/lucene/analysis/CharTokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/CharTokenizer.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/CharTokenizer.java (working copy)
@@ -114,18 +114,6 @@
offsetAtt.setOffset(finalOffset, finalOffset);
}
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next(final Token reusableToken) throws IOException {
- return super.next(reusableToken);
- }
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next() throws IOException {
- return super.next();
- }
-
public void reset(Reader input) throws IOException {
super.reset(input);
bufferIndex = 0;
Index: src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
===================================================================
--- src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java (working copy)
@@ -1,269 +0,0 @@
-package org.apache.lucene.analysis;
-
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * A filter that replaces accented characters in the ISO Latin 1 character set
- * (ISO-8859-1) by their unaccented equivalent. The case will not be altered.
- *
- * For instance, 'à' will be replaced by 'a'.
- *
- *
- * @deprecated in favor of {@link ASCIIFoldingFilter} which covers a superset
- * of Latin 1. This class will be removed in Lucene 3.0.
- */
-public class ISOLatin1AccentFilter extends TokenFilter {
- public ISOLatin1AccentFilter(TokenStream input) {
- super(input);
- termAtt = addAttribute(TermAttribute.class);
- }
-
- private char[] output = new char[256];
- private int outputPos;
- private TermAttribute termAtt;
-
- public final boolean incrementToken() throws java.io.IOException {
- if (input.incrementToken()) {
- final char[] buffer = termAtt.termBuffer();
- final int length = termAtt.termLength();
- // If no characters actually require rewriting then we
- // just return token as-is:
- for(int i=0;i= '\u00c0' && c <= '\uFB06') {
- removeAccents(buffer, length);
- termAtt.setTermBuffer(output, 0, outputPos);
- break;
- }
- }
- return true;
- } else
- return false;
- }
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next(final Token reusableToken) throws java.io.IOException {
- return super.next(reusableToken);
- }
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next() throws java.io.IOException {
- return super.next();
- }
-
- /**
- * To replace accented characters in a String by unaccented equivalents.
- */
- public final void removeAccents(char[] input, int length) {
-
- // Worst-case length required:
- final int maxSizeNeeded = 2*length;
-
- int size = output.length;
- while (size < maxSizeNeeded)
- size *= 2;
-
- if (size != output.length)
- output = new char[size];
-
- outputPos = 0;
-
- int pos = 0;
-
- for (int i=0; i '\uFB06')
- output[outputPos++] = c;
- else {
- switch (c) {
- case '\u00C0' : // À
- case '\u00C1' : // Á
- case '\u00C2' : // Â
- case '\u00C3' : // Ã
- case '\u00C4' : // Ä
- case '\u00C5' : // Å
- output[outputPos++] = 'A';
- break;
- case '\u00C6' : // Æ
- output[outputPos++] = 'A';
- output[outputPos++] = 'E';
- break;
- case '\u00C7' : // Ç
- output[outputPos++] = 'C';
- break;
- case '\u00C8' : // È
- case '\u00C9' : // É
- case '\u00CA' : // Ê
- case '\u00CB' : // Ë
- output[outputPos++] = 'E';
- break;
- case '\u00CC' : // Ì
- case '\u00CD' : // Í
- case '\u00CE' : // Î
- case '\u00CF' : // Ï
- output[outputPos++] = 'I';
- break;
- case '\u0132' : // IJ
- output[outputPos++] = 'I';
- output[outputPos++] = 'J';
- break;
- case '\u00D0' : // Ð
- output[outputPos++] = 'D';
- break;
- case '\u00D1' : // Ñ
- output[outputPos++] = 'N';
- break;
- case '\u00D2' : // Ò
- case '\u00D3' : // Ó
- case '\u00D4' : // Ô
- case '\u00D5' : // Õ
- case '\u00D6' : // Ö
- case '\u00D8' : // Ø
- output[outputPos++] = 'O';
- break;
- case '\u0152' : // Œ
- output[outputPos++] = 'O';
- output[outputPos++] = 'E';
- break;
- case '\u00DE' : // Þ
- output[outputPos++] = 'T';
- output[outputPos++] = 'H';
- break;
- case '\u00D9' : // Ù
- case '\u00DA' : // Ú
- case '\u00DB' : // Û
- case '\u00DC' : // Ü
- output[outputPos++] = 'U';
- break;
- case '\u00DD' : // Ý
- case '\u0178' : // Ÿ
- output[outputPos++] = 'Y';
- break;
- case '\u00E0' : // à
- case '\u00E1' : // á
- case '\u00E2' : // â
- case '\u00E3' : // ã
- case '\u00E4' : // ä
- case '\u00E5' : // å
- output[outputPos++] = 'a';
- break;
- case '\u00E6' : // æ
- output[outputPos++] = 'a';
- output[outputPos++] = 'e';
- break;
- case '\u00E7' : // ç
- output[outputPos++] = 'c';
- break;
- case '\u00E8' : // è
- case '\u00E9' : // é
- case '\u00EA' : // ê
- case '\u00EB' : // ë
- output[outputPos++] = 'e';
- break;
- case '\u00EC' : // ì
- case '\u00ED' : // í
- case '\u00EE' : // î
- case '\u00EF' : // ï
- output[outputPos++] = 'i';
- break;
- case '\u0133' : // ij
- output[outputPos++] = 'i';
- output[outputPos++] = 'j';
- break;
- case '\u00F0' : // ð
- output[outputPos++] = 'd';
- break;
- case '\u00F1' : // ñ
- output[outputPos++] = 'n';
- break;
- case '\u00F2' : // ò
- case '\u00F3' : // ó
- case '\u00F4' : // ô
- case '\u00F5' : // õ
- case '\u00F6' : // ö
- case '\u00F8' : // ø
- output[outputPos++] = 'o';
- break;
- case '\u0153' : // œ
- output[outputPos++] = 'o';
- output[outputPos++] = 'e';
- break;
- case '\u00DF' : // ß
- output[outputPos++] = 's';
- output[outputPos++] = 's';
- break;
- case '\u00FE' : // þ
- output[outputPos++] = 't';
- output[outputPos++] = 'h';
- break;
- case '\u00F9' : // ù
- case '\u00FA' : // ú
- case '\u00FB' : // û
- case '\u00FC' : // ü
- output[outputPos++] = 'u';
- break;
- case '\u00FD' : // ý
- case '\u00FF' : // ÿ
- output[outputPos++] = 'y';
- break;
- case '\uFB00': // ff
- output[outputPos++] = 'f';
- output[outputPos++] = 'f';
- break;
- case '\uFB01': // fi
- output[outputPos++] = 'f';
- output[outputPos++] = 'i';
- break;
- case '\uFB02': // fl
- output[outputPos++] = 'f';
- output[outputPos++] = 'l';
- break;
- // following 2 are commented as they can break the maxSizeNeeded (and doing *3 could be expensive)
-// case '\uFB03': // ffi
-// output[outputPos++] = 'f';
-// output[outputPos++] = 'f';
-// output[outputPos++] = 'i';
-// break;
-// case '\uFB04': // ffl
-// output[outputPos++] = 'f';
-// output[outputPos++] = 'f';
-// output[outputPos++] = 'l';
-// break;
- case '\uFB05': // ſt
- output[outputPos++] = 'f';
- output[outputPos++] = 't';
- break;
- case '\uFB06': // st
- output[outputPos++] = 's';
- output[outputPos++] = 't';
- break;
- default :
- output[outputPos++] = c;
- break;
- }
- }
- }
- }
-}
Index: src/java/org/apache/lucene/analysis/KeywordTokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/KeywordTokenizer.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/KeywordTokenizer.java (working copy)
@@ -27,7 +27,7 @@
/**
* Emits the entire input as a single token.
*/
-public class KeywordTokenizer extends Tokenizer {
+public final class KeywordTokenizer extends Tokenizer {
private static final int DEFAULT_BUFFER_SIZE = 256;
@@ -88,18 +88,6 @@
offsetAtt.setOffset(finalOffset, finalOffset);
}
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next(final Token reusableToken) throws IOException {
- return super.next(reusableToken);
- }
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next() throws IOException {
- return super.next();
- }
-
public void reset(Reader input) throws IOException {
super.reset(input);
this.done = false;
Index: src/java/org/apache/lucene/analysis/SinkTokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/SinkTokenizer.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/SinkTokenizer.java (working copy)
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-
-/**
- * A SinkTokenizer can be used to cache Tokens for use in an Analyzer
- *
- * WARNING: {@link TeeTokenFilter} and {@link SinkTokenizer} only work with the old TokenStream API.
- * If you switch to the new API, you need to use {@link TeeSinkTokenFilter} instead, which offers
- * the same functionality.
- * @see TeeTokenFilter
- * @deprecated Use {@link TeeSinkTokenFilter} instead
- *
- **/
-public class SinkTokenizer extends Tokenizer {
- protected List/**/ lst = new ArrayList/**/();
- protected Iterator/**/ iter;
-
- public SinkTokenizer(List/**/ input) {
- this.lst = input;
- if (this.lst == null) this.lst = new ArrayList/**/();
- }
-
- public SinkTokenizer() {
- this.lst = new ArrayList/**/();
- }
-
- public SinkTokenizer(int initCap){
- this.lst = new ArrayList/**/(initCap);
- }
-
- /**
- * Get the tokens in the internal List.
- *
- * WARNING: Adding tokens to this list requires the {@link #reset()} method to be called in order for them
- * to be made available. Also, this Tokenizer does nothing to protect against {@link java.util.ConcurrentModificationException}s
- * in the case of adds happening while {@link #next(org.apache.lucene.analysis.Token)} is being called.
- *
- * WARNING: Since this SinkTokenizer can be reset and the cached tokens made available again, do not modify them. Modify clones instead.
- *
- * @return A List of {@link org.apache.lucene.analysis.Token}s
- */
- public List/**/ getTokens() {
- return lst;
- }
-
- /**
- * Returns the next token out of the list of cached tokens
- * @return The next {@link org.apache.lucene.analysis.Token} in the Sink.
- * @throws IOException
- */
- public Token next(final Token reusableToken) throws IOException {
- assert reusableToken != null;
- if (iter == null) iter = lst.iterator();
- // Since this TokenStream can be reset we have to maintain the tokens as immutable
- if (iter.hasNext()) {
- Token nextToken = (Token) iter.next();
- return (Token) nextToken.clone();
- }
- return null;
- }
-
- /**
- * Override this method to cache only certain tokens, or new tokens based
- * on the old tokens.
- *
- * @param t The {@link org.apache.lucene.analysis.Token} to add to the sink
- */
- public void add(Token t) {
- if (t == null) return;
- lst.add((Token) t.clone());
- }
-
- public void close() throws IOException {
- //nothing to close
- input = null;
- lst = null;
- }
-
- /**
- * Reset the internal data structures to the start at the front of the list of tokens. Should be called
- * if tokens were added to the list after an invocation of {@link #next(Token)}
- * @throws IOException
- */
- public void reset() throws IOException {
- iter = lst.iterator();
- }
-}
-
Index: src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (working copy)
@@ -45,7 +45,7 @@
* directory to your project and maintaining your own grammar-based tokenizer.
*/
-public class StandardTokenizer extends Tokenizer {
+public final class StandardTokenizer extends Tokenizer {
/** A private instance of the JFlex-constructed scanner */
private final StandardTokenizerImpl scanner;
@@ -209,18 +209,6 @@
offsetAtt.setOffset(finalOffset, finalOffset);
}
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next(final Token reusableToken) throws IOException {
- return super.next(reusableToken);
- }
-
- /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
- * not be overridden. Delegates to the backwards compatibility layer. */
- public final Token next() throws IOException {
- return super.next();
- }
-
/*
* (non-Javadoc)
*
Index: src/java/org/apache/lucene/analysis/TeeTokenFilter.java
===================================================================
--- src/java/org/apache/lucene/analysis/TeeTokenFilter.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/TeeTokenFilter.java (working copy)
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis;
-
-import java.io.IOException;
-
-
-/**
- * Works in conjunction with the SinkTokenizer to provide the ability to set aside tokens
- * that have already been analyzed. This is useful in situations where multiple fields share
- * many common analysis steps and then go their separate ways.
- *
- * It is also useful for doing things like entity extraction or proper noun analysis as
- * part of the analysis workflow and saving off those tokens for use in another field.
- *
- *
-SinkTokenizer sink1 = new SinkTokenizer();
-SinkTokenizer sink2 = new SinkTokenizer();
-
-TokenStream source1 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(reader1), sink1), sink2);
-TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(reader2), sink1), sink2);
-
-TokenStream final1 = new LowerCaseFilter(source1);
-TokenStream final2 = source2;
-TokenStream final3 = new EntityDetect(sink1);
-TokenStream final4 = new URLDetect(sink2);
-
-d.add(new Field("f1", final1));
-d.add(new Field("f2", final2));
-d.add(new Field("f3", final3));
-d.add(new Field("f4", final4));
- *
- * In this example, sink1 and sink2 will both get tokens from both
- * reader1 and reader2 after whitespace tokenizer
- * and now we can further wrap any of these in extra analysis, and more "sources" can be inserted if desired.
- * It is important, that tees are consumed before sinks (in the above example, the field names must be
- * less the sink's field names).
- * Note, the EntityDetect and URLDetect TokenStreams are for the example and do not currently exist in Lucene
-
- *
- * See LUCENE-1058.
- *
- * WARNING: {@link TeeTokenFilter} and {@link SinkTokenizer} only work with the old TokenStream API.
- * If you switch to the new API, you need to use {@link TeeSinkTokenFilter} instead, which offers
- * the same functionality.
-
- * @see SinkTokenizer
- * @deprecated Use {@link TeeSinkTokenFilter} instead
- **/
-public class TeeTokenFilter extends TokenFilter {
- SinkTokenizer sink;
-
- public TeeTokenFilter(TokenStream input, SinkTokenizer sink) {
- super(input);
- this.sink = sink;
- }
-
- public Token next(final Token reusableToken) throws IOException {
- assert reusableToken != null;
- Token nextToken = input.next(reusableToken);
- sink.add(nextToken);
- return nextToken;
- }
-
-}
Index: src/java/org/apache/lucene/analysis/Token.java
===================================================================
--- src/java/org/apache/lucene/analysis/Token.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/Token.java (working copy)
@@ -867,9 +867,6 @@
if (payload !=null) {
to.payload = (Payload) payload.clone();
}
- // remove the following optimization in 3.0 when old TokenStream API removed:
- } else if (target instanceof TokenWrapper) {
- ((TokenWrapper) target).delegate = (Token) this.clone();
} else {
initTermBuffer();
((TermAttribute) target).setTermBuffer(termBuffer, 0, termLength);
Index: src/java/org/apache/lucene/analysis/TokenStream.java
===================================================================
--- src/java/org/apache/lucene/analysis/TokenStream.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/TokenStream.java (working copy)
@@ -18,18 +18,10 @@
*/
import java.io.IOException;
-import java.util.IdentityHashMap;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Payload;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeSource;
@@ -78,103 +70,19 @@
* You can find some example code for the new API in the analysis package level
* Javadoc.
*
- * Sometimes it is desirable to capture a current state of a TokenStream
- * , e. g. for buffering purposes (see {@link CachingTokenFilter},
+ * Sometimes it is desirable to capture a current state of a TokenStream,
+ * e.g. for buffering purposes (see {@link CachingTokenFilter},
* {@link TeeSinkTokenFilter}). For this usecase
* {@link AttributeSource#captureState} and {@link AttributeSource#restoreState}
* can be used.
*/
public abstract class TokenStream extends AttributeSource {
- /** @deprecated Remove this when old API is removed! */
- private static final AttributeFactory DEFAULT_TOKEN_WRAPPER_ATTRIBUTE_FACTORY
- = new TokenWrapperAttributeFactory(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
-
- /** @deprecated Remove this when old API is removed! */
- private final TokenWrapper tokenWrapper;
-
- /** @deprecated Remove this when old API is removed! */
- private static boolean onlyUseNewAPI = false;
-
- /** @deprecated Remove this when old API is removed! */
- private final MethodSupport supportedMethods = getSupportedMethods(this.getClass());
-
- /** @deprecated Remove this when old API is removed! */
- private static final class MethodSupport {
- final boolean hasIncrementToken, hasReusableNext, hasNext;
-
- MethodSupport(Class clazz) {
- hasIncrementToken = isMethodOverridden(clazz, "incrementToken");
- hasReusableNext = isMethodOverridden(clazz, "next", Token.class);
- hasNext = isMethodOverridden(clazz, "next");
- }
-
- private static boolean isMethodOverridden(Class clazz, String name, Class... params) {
- try {
- return clazz.getMethod(name, params).getDeclaringClass() != TokenStream.class;
- } catch (NoSuchMethodException e) {
- // should not happen
- throw new RuntimeException(e);
- }
- }
- }
-
- /** @deprecated Remove this when old API is removed! */
- private static final IdentityHashMap,MethodSupport> knownMethodSupport =
- new IdentityHashMap,MethodSupport>();
-
- /** @deprecated Remove this when old API is removed! */
- private static MethodSupport getSupportedMethods(Class clazz) {
- MethodSupport supportedMethods;
- synchronized(knownMethodSupport) {
- supportedMethods = knownMethodSupport.get(clazz);
- if (supportedMethods == null) {
- knownMethodSupport.put(clazz, supportedMethods = new MethodSupport(clazz));
- }
- }
- return supportedMethods;
- }
-
- /** @deprecated Remove this when old API is removed! */
- private static final class TokenWrapperAttributeFactory extends AttributeFactory {
- private final AttributeFactory delegate;
-
- private TokenWrapperAttributeFactory(AttributeFactory delegate) {
- this.delegate = delegate;
- }
-
- public AttributeImpl createAttributeInstance(Class attClass) {
- return attClass.isAssignableFrom(TokenWrapper.class)
- ? new TokenWrapper()
- : delegate.createAttributeInstance(attClass);
- }
-
- // this is needed for TeeSinkTokenStream's check for compatibility of AttributeSource,
- // so two TokenStreams using old API have the same AttributeFactory wrapped by this one.
- public boolean equals(Object other) {
- if (this == other) return true;
- if (other instanceof TokenWrapperAttributeFactory) {
- final TokenWrapperAttributeFactory af = (TokenWrapperAttributeFactory) other;
- return this.delegate.equals(af.delegate);
- }
- return false;
- }
-
- public int hashCode() {
- return delegate.hashCode() ^ 0x0a45ff31;
- }
- }
-
/**
* A TokenStream using the default attribute factory.
*/
protected TokenStream() {
- super(onlyUseNewAPI
- ? AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY
- : TokenStream.DEFAULT_TOKEN_WRAPPER_ATTRIBUTE_FACTORY
- );
- tokenWrapper = initTokenWrapper(null);
- check();
+ super();
}
/**
@@ -182,105 +90,16 @@
*/
protected TokenStream(AttributeSource input) {
super(input);
- tokenWrapper = initTokenWrapper(input);
- check();
}
/**
* A TokenStream using the supplied AttributeFactory for creating new {@link Attribute} instances.
*/
protected TokenStream(AttributeFactory factory) {
- super(onlyUseNewAPI
- ? factory
- : new TokenWrapperAttributeFactory(factory)
- );
- tokenWrapper = initTokenWrapper(null);
- check();
+ super(factory);
}
-
- /** @deprecated Remove this when old API is removed! */
- private TokenWrapper initTokenWrapper(AttributeSource input) {
- if (onlyUseNewAPI) {
- // no wrapper needed
- return null;
- } else {
- // if possible get the wrapper from the filter's input stream
- if (input instanceof TokenStream && ((TokenStream) input).tokenWrapper != null) {
- return ((TokenStream) input).tokenWrapper;
- }
- // check that all attributes are implemented by the same TokenWrapper instance
- final Attribute att = addAttribute(TermAttribute.class);
- if (att instanceof TokenWrapper &&
- addAttribute(TypeAttribute.class) == att &&
- addAttribute(PositionIncrementAttribute.class) == att &&
- addAttribute(FlagsAttribute.class) == att &&
- addAttribute(OffsetAttribute.class) == att &&
- addAttribute(PayloadAttribute.class) == att
- ) {
- return (TokenWrapper) att;
- } else {
- throw new UnsupportedOperationException(
- "If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class "+
- "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been "+
- "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!"
- );
- }
- }
- }
-
- /** @deprecated Remove this when old API is removed! */
- private void check() {
- if (onlyUseNewAPI && !supportedMethods.hasIncrementToken) {
- throw new UnsupportedOperationException(getClass().getName()+" does not implement incrementToken() which is needed for onlyUseNewAPI.");
- }
-
- // a TokenStream subclass must at least implement one of the methods!
- if (!(supportedMethods.hasIncrementToken || supportedMethods.hasNext || supportedMethods.hasReusableNext)) {
- throw new UnsupportedOperationException(getClass().getName()+" does not implement any of incrementToken(), next(Token), next().");
- }
- }
/**
- * For extra performance you can globally enable the new
- * {@link #incrementToken} API using {@link Attribute}s. There will be a
- * small, but in most cases negligible performance increase by enabling this,
- * but it only works if all TokenStreams use the new API and
- * implement {@link #incrementToken}. This setting can only be enabled
- * globally.
- *
- * This setting only affects TokenStreams instantiated after this
- * call. All TokenStreams already created use the other setting.
- *
- * All core {@link Analyzer}s are compatible with this setting, if you have
- * your own TokenStreams that are also compatible, you should enable
- * this.
- *
- * When enabled, tokenization may throw {@link UnsupportedOperationException}
- * s, if the whole tokenizer chain is not compatible eg one of the
- * TokenStreams does not implement the new TokenStream API.
- *
- * The default is false, so there is the fallback to the old API
- * available.
- *
- * @deprecated This setting will no longer be needed in Lucene 3.0 as the old
- * API will be removed.
- */
- public static void setOnlyUseNewAPI(boolean onlyUseNewAPI) {
- TokenStream.onlyUseNewAPI = onlyUseNewAPI;
- }
-
- /**
- * Returns if only the new API is used.
- *
- * @see #setOnlyUseNewAPI
- * @deprecated This setting will no longer be needed in Lucene 3.0 as
- * the old API will be removed.
- */
- public static boolean getOnlyUseNewAPI() {
- return onlyUseNewAPI;
- }
-
- /**
* Consumers (ie {@link IndexWriter}) use this method to advance the stream to
* the next token. Implementing classes must implement this method and update
* the appropriate {@link AttributeImpl}s with the attributes of the next
@@ -293,7 +112,7 @@
*
* This method is called for every token of a document, so an efficient
* implementation is crucial for good performance. To avoid calls to
- * {@link #addAttribute(Class)} and {@link #getAttribute(Class)} or downcasts,
+ * {@link #addAttribute(Class)} and {@link #getAttribute(Class)},
* references to all {@link AttributeImpl}s that this stream uses should be
* retrieved during instantiation.
*
@@ -303,25 +122,8 @@
* {@link #incrementToken()}.
*
* @return false for end of stream; true otherwise
- *
- *
- * Note that this method will be defined abstract in Lucene
- * 3.0.
*/
- public boolean incrementToken() throws IOException {
- assert tokenWrapper != null;
-
- final Token token;
- if (supportedMethods.hasReusableNext) {
- token = next(tokenWrapper.delegate);
- } else {
- assert supportedMethods.hasNext;
- token = next();
- }
- if (token == null) return false;
- tokenWrapper.delegate = token;
- return true;
- }
+ public abstract boolean incrementToken() throws IOException;
/**
* This method is called by the consumer after the last token has been
@@ -341,83 +143,6 @@
}
/**
- * Returns the next token in the stream, or null at EOS. When possible, the
- * input Token should be used as the returned Token (this gives fastest
- * tokenization performance), but this is not required and a new Token may be
- * returned. Callers may re-use a single Token instance for successive calls
- * to this method.
- *
- * This implicitly defines a "contract" between consumers (callers of this
- * method) and producers (implementations of this method that are the source
- * for tokens):
- *
- * - A consumer must fully consume the previously returned {@link Token}
- * before calling this method again.
- * - A producer must call {@link Token#clear()} before setting the fields in
- * it and returning it
- *
- * Also, the producer must make no assumptions about a {@link Token} after it
- * has been returned: the caller may arbitrarily change it. If the producer
- * needs to hold onto the {@link Token} for subsequent calls, it must clone()
- * it before storing it. Note that a {@link TokenFilter} is considered a
- * consumer.
- *
- * @param reusableToken a {@link Token} that may or may not be used to return;
- * this parameter should never be null (the callee is not required to
- * check for null before using it, but it is a good idea to assert that
- * it is not null.)
- * @return next {@link Token} in the stream or null if end-of-stream was hit
- * @deprecated The new {@link #incrementToken()} and {@link AttributeSource}
- * APIs should be used instead.
- */
- public Token next(final Token reusableToken) throws IOException {
- assert reusableToken != null;
-
- if (tokenWrapper == null)
- throw new UnsupportedOperationException("This TokenStream only supports the new Attributes API.");
-
- if (supportedMethods.hasIncrementToken) {
- tokenWrapper.delegate = reusableToken;
- return incrementToken() ? tokenWrapper.delegate : null;
- } else {
- assert supportedMethods.hasNext;
- return next();
- }
- }
-
- /**
- * Returns the next {@link Token} in the stream, or null at EOS.
- *
- * @deprecated The returned Token is a "full private copy" (not re-used across
- * calls to {@link #next()}) but will be slower than calling
- * {@link #next(Token)} or using the new {@link #incrementToken()}
- * method with the new {@link AttributeSource} API.
- */
- public Token next() throws IOException {
- if (tokenWrapper == null)
- throw new UnsupportedOperationException("This TokenStream only supports the new Attributes API.");
-
- final Token nextToken;
- if (supportedMethods.hasIncrementToken) {
- final Token savedDelegate = tokenWrapper.delegate;
- tokenWrapper.delegate = new Token();
- nextToken = incrementToken() ? tokenWrapper.delegate : null;
- tokenWrapper.delegate = savedDelegate;
- } else {
- assert supportedMethods.hasReusableNext;
- nextToken = next(new Token());
- }
-
- if (nextToken != null) {
- Payload p = nextToken.getPayload();
- if (p != null) {
- nextToken.setPayload((Payload) p.clone());
- }
- }
- return nextToken;
- }
-
- /**
* Resets this stream to the beginning. This is an optional operation, so
* subclasses may or may not implement this method. {@link #reset()} is not needed for
* the standard indexing process. However, if the tokens of a
Index: src/java/org/apache/lucene/analysis/TokenWrapper.java
===================================================================
--- src/java/org/apache/lucene/analysis/TokenWrapper.java (revision 824063)
+++ src/java/org/apache/lucene/analysis/TokenWrapper.java (working copy)
@@ -1,166 +0,0 @@
-package org.apache.lucene.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.index.Payload;
-import org.apache.lucene.util.AttributeImpl;
-
-/**
- * This class wraps a Token and supplies a single attribute instance
- * where the delegate token can be replaced.
- * @deprecated Will be removed, when old TokenStream API is removed.
- */
-final class TokenWrapper extends AttributeImpl
- implements Cloneable, TermAttribute, TypeAttribute, PositionIncrementAttribute,
- FlagsAttribute, OffsetAttribute, PayloadAttribute {
-
- Token delegate;
-
- TokenWrapper() {
- this(new Token());
- }
-
- TokenWrapper(Token delegate) {
- this.delegate = delegate;
- }
-
- // TermAttribute:
-
- public String term() {
- return delegate.term();
- }
-
- public void setTermBuffer(char[] buffer, int offset, int length) {
- delegate.setTermBuffer(buffer, offset, length);
- }
-
- public void setTermBuffer(String buffer) {
- delegate.setTermBuffer(buffer);
- }
-
- public void setTermBuffer(String buffer, int offset, int length) {
- delegate.setTermBuffer(buffer, offset, length);
- }
-
- public char[] termBuffer() {
- return delegate.termBuffer();
- }
-
- public char[] resizeTermBuffer(int newSize) {
- return delegate.resizeTermBuffer(newSize);
- }
-
- public int termLength() {
- return delegate.termLength();
- }
-
- public void setTermLength(int length) {
- delegate.setTermLength(length);
- }
-
- // TypeAttribute:
-
- public String type() {
- return delegate.type();
- }
-
- public void setType(String type) {
- delegate.setType(type);
- }
-
- public void setPositionIncrement(int positionIncrement) {
- delegate.setPositionIncrement(positionIncrement);
- }
-
- public int getPositionIncrement() {
- return delegate.getPositionIncrement();
- }
-
- // FlagsAttribute
-
- public int getFlags() {
- return delegate.getFlags();
- }
-
- public void setFlags(int flags) {
- delegate.setFlags(flags);
- }
-
- // OffsetAttribute
-
- public int startOffset() {
- return delegate.startOffset();
- }
-
- public void setOffset(int startOffset, int endOffset) {
- delegate.setOffset(startOffset, endOffset);
- }
-
- public int endOffset() {
- return delegate.endOffset();
- }
-
- // PayloadAttribute
-
- public Payload getPayload() {
- return delegate.getPayload();
- }
-
- public void setPayload(Payload payload) {
- delegate.setPayload(payload);
- }
-
- // AttributeImpl
-
- public void clear() {
- delegate.clear();
- }
-
- public String toString() {
- return delegate.toString();
- }
-
- public int hashCode() {
- return delegate.hashCode();
- }
-
- public boolean equals(Object other) {
- if (other instanceof TokenWrapper) {
- return ((TokenWrapper) other).delegate.equals(this.delegate);
- }
- return false;
- }
-
- public Object clone() {
- return new TokenWrapper((Token) delegate.clone());
- }
-
- public void copyTo(AttributeImpl target) {
- if (target instanceof TokenWrapper) {
- ((TokenWrapper) target).delegate = (Token) this.delegate.clone();
- } else {
- this.delegate.copyTo(target);
- }
- }
-}
Index: src/test/org/apache/lucene/analysis/TestToken.java
===================================================================
--- src/test/org/apache/lucene/analysis/TestToken.java (revision 824063)
+++ src/test/org/apache/lucene/analysis/TestToken.java (working copy)
@@ -18,9 +18,13 @@
*/
import org.apache.lucene.index.Payload;
-import org.apache.lucene.analysis.tokenattributes.TestSimpleAttributeImpls;
+import org.apache.lucene.analysis.tokenattributes.*;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import java.io.StringReader;
+
public class TestToken extends LuceneTestCase {
public TestToken(String name) {
@@ -219,4 +223,33 @@
assertEquals(pl, copy.getPayload());
assertNotSame(pl, copy.getPayload());
}
+
+ public interface SenselessAttribute extends Attribute {}
+
+ public static final class SenselessAttributeImpl extends AttributeImpl implements SenselessAttribute {
+ public void copyTo(AttributeImpl target) {}
+ public void clear() {}
+ public boolean equals(Object o) { return (o instanceof SenselessAttributeImpl); }
+ public int hashCode() { return 0; }
+ }
+
+ public void testTokenAttributeFactory() throws Exception {
+ TokenStream ts = new WhitespaceTokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, new StringReader("foo bar"));
+
+ assertTrue("TypeAttribute is not implemented by SenselessAttributeImpl",
+ ts.addAttribute(SenselessAttribute.class) instanceof SenselessAttributeImpl);
+
+ assertTrue("TermAttribute is not implemented by Token",
+ ts.addAttribute(TermAttribute.class) instanceof Token);
+ assertTrue("OffsetAttribute is not implemented by Token",
+ ts.addAttribute(OffsetAttribute.class) instanceof Token);
+ assertTrue("FlagsAttribute is not implemented by Token",
+ ts.addAttribute(FlagsAttribute.class) instanceof Token);
+ assertTrue("PayloadAttribute is not implemented by Token",
+ ts.addAttribute(PayloadAttribute.class) instanceof Token);
+ assertTrue("PositionIncrementAttribute is not implemented by Token",
+ ts.addAttribute(PositionIncrementAttribute.class) instanceof Token);
+ assertTrue("TypeAttribute is not implemented by Token",
+ ts.addAttribute(TypeAttribute.class) instanceof Token);
+ }
}
Index: src/test/org/apache/lucene/util/TestAttributeSource.java
===================================================================
--- src/test/org/apache/lucene/util/TestAttributeSource.java (revision 824063)
+++ src/test/org/apache/lucene/util/TestAttributeSource.java (working copy)
@@ -125,4 +125,20 @@
assertEquals("Token should only printed once", "("+tok.toString()+")", src.toString());
}
+ public void testDefaultAttributeFactory() throws Exception {
+ AttributeSource src = new AttributeSource();
+
+ assertTrue("TermAttribute is not implemented by TermAttributeImpl",
+ src.addAttribute(TermAttribute.class) instanceof TermAttributeImpl);
+ assertTrue("OffsetAttribute is not implemented by OffsetAttributeImpl",
+ src.addAttribute(OffsetAttribute.class) instanceof OffsetAttributeImpl);
+ assertTrue("FlagsAttribute is not implemented by FlagsAttributeImpl",
+ src.addAttribute(FlagsAttribute.class) instanceof FlagsAttributeImpl);
+ assertTrue("PayloadAttribute is not implemented by PayloadAttributeImpl",
+ src.addAttribute(PayloadAttribute.class) instanceof PayloadAttributeImpl);
+ assertTrue("PositionIncrementAttribute is not implemented by PositionIncrementAttributeImpl",
+ src.addAttribute(PositionIncrementAttribute.class) instanceof PositionIncrementAttributeImpl);
+ assertTrue("TypeAttribute is not implemented by TypeAttributeImpl",
+ src.addAttribute(TypeAttribute.class) instanceof TypeAttributeImpl);
+ }
}