Index: lucene/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt =================================================================== --- lucene/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt (revision 945106) +++ lucene/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt (working copy) @@ -17,4 +17,5 @@ WARNING: if you change StandardTokenizerImpl*.jflex and need to regenerate - the tokenizer, only use the trunk version of JFlex 1.5 at the moment! + the tokenizer, only use the trunk version of JFlex 1.5 (with a minimum + SVN revision 591) at the moment! Index: lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java =================================================================== --- lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (revision 945106) +++ lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (working copy) @@ -201,7 +201,7 @@ @Override public void reset(Reader reader) throws IOException { super.reset(reader); - scanner.reset(reader); + scanner.yyreset(reader); } /** Index: lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java =================================================================== --- lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java (revision 945106) +++ lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java (working copy) @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 13:07 */ +/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:50 */ package org.apache.lucene.analysis.standard; @@ -33,8 +33,8 @@ /** * This class is a scanner generated by * JFlex 1.5.0-SNAPSHOT - * on 10.04.10 13:07 from the specification file - * C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex + * on 17.05.10 14:50 from the specification file + * C:/Users/Uwe Schindler/Projects/lucene/newtrunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex */ class StandardTokenizerImpl31 implements StandardTokenizerInterface { @@ -379,19 +379,8 @@ t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); } -/** - * Resets the Tokenizer to a new Reader. - */ -public final void reset(Reader r) { - // reset to default buffer size, if buffer has grown - if (zzBuffer.length > ZZ_BUFFERSIZE) { - zzBuffer = new char[ZZ_BUFFERSIZE]; - } - yyreset(r); -} - /** * Creates a new scanner * There is also a java.io.InputStream version of this constructor. @@ -505,6 +494,8 @@ * cannot be reused (internal buffer is discarded and lost). * Lexical state is set to ZZ_INITIAL. * + * Internal scan buffer is resized down to its initial length, if it has grown. + * * @param reader the new input stream */ public final void yyreset(java.io.Reader reader) { @@ -516,6 +507,8 @@ zzCurrentPos = zzMarkedPos = 0; yyline = yychar = yycolumn = 0; zzLexicalState = YYINITIAL; + if (zzBuffer.length > ZZ_BUFFERSIZE) + zzBuffer = new char[ZZ_BUFFERSIZE]; } Index: lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex =================================================================== --- lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex (revision 945106) +++ lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex (working copy) @@ -67,17 +67,6 @@ t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); } -/** - * Resets the Tokenizer to a new Reader. - */ -public final void reset(Reader r) { - // reset to default buffer size, if buffer has grown - if (zzBuffer.length > ZZ_BUFFERSIZE) { - zzBuffer = new char[ZZ_BUFFERSIZE]; - } - yyreset(r); -} - %} THAI = [\u0E00-\u0E59] Index: lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java =================================================================== --- lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java (revision 945106) +++ lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java (working copy) @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 13:07 */ +/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:50 */ package org.apache.lucene.analysis.standard; @@ -33,8 +33,8 @@ /** * This class is a scanner generated by * JFlex 1.5.0-SNAPSHOT - * on 10.04.10 13:07 from the specification file - * C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex + * on 17.05.10 14:50 from the specification file + * C:/Users/Uwe Schindler/Projects/lucene/newtrunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex */ class StandardTokenizerImplOrig implements StandardTokenizerInterface { @@ -375,19 +375,8 @@ t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); } -/** - * Resets the Tokenizer to a new Reader. - */ -public final void reset(Reader r) { - // reset to default buffer size, if buffer has grown - if (zzBuffer.length > ZZ_BUFFERSIZE) { - zzBuffer = new char[ZZ_BUFFERSIZE]; - } - yyreset(r); -} - /** * Creates a new scanner * There is also a java.io.InputStream version of this constructor. @@ -501,6 +490,8 @@ * cannot be reused (internal buffer is discarded and lost). * Lexical state is set to ZZ_INITIAL. * + * Internal scan buffer is resized down to its initial length, if it has grown. + * * @param reader the new input stream */ public final void yyreset(java.io.Reader reader) { @@ -512,6 +503,8 @@ zzCurrentPos = zzMarkedPos = 0; yyline = yychar = yycolumn = 0; zzLexicalState = YYINITIAL; + if (zzBuffer.length > ZZ_BUFFERSIZE) + zzBuffer = new char[ZZ_BUFFERSIZE]; } Index: lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex =================================================================== --- lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex (revision 945106) +++ lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex (working copy) @@ -67,17 +67,6 @@ t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); } -/** - * Resets the Tokenizer to a new Reader. - */ -public final void reset(Reader r) { - // reset to default buffer size, if buffer has grown - if (zzBuffer.length > ZZ_BUFFERSIZE) { - zzBuffer = new char[ZZ_BUFFERSIZE]; - } - yyreset(r); -} - %} THAI = [\u0E00-\u0E59] Index: lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java =================================================================== --- lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java (revision 945106) +++ lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java (working copy) @@ -47,7 +47,7 @@ * * @param reader the new input stream */ - void reset(Reader reader); + void yyreset(Reader reader); /** * Returns the length of the matched text region. Index: modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java =================================================================== --- modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java (revision 945106) +++ modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java (working copy) @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 12:56 */ +/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:51 */ package org.apache.lucene.analysis.wikipedia; @@ -25,8 +25,8 @@ /** * This class is a scanner generated by * JFlex 1.5.0-SNAPSHOT - * on 10.04.10 12:56 from the specification file - * C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex + * on 17.05.10 14:51 from the specification file + * C:/Users/Uwe Schindler/Projects/lucene/newtrunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex */ class WikipediaTokenizerImpl { @@ -613,6 +613,8 @@ * cannot be reused (internal buffer is discarded and lost). * Lexical state is set to ZZ_INITIAL. * + * Internal scan buffer is resized down to its initial length, if it has grown. + * * @param reader the new input stream */ public final void yyreset(java.io.Reader reader) { @@ -624,6 +626,8 @@ zzCurrentPos = zzMarkedPos = 0; yyline = yychar = yycolumn = 0; zzLexicalState = YYINITIAL; + if (zzBuffer.length > ZZ_BUFFERSIZE) + zzBuffer = new char[ZZ_BUFFERSIZE]; }