Index: BUILD.txt
===================================================================
RCS file: /home/cvspublic/jakarta-lucene/BUILD.txt,v
retrieving revision 1.6
diff -u -r1.6 BUILD.txt
--- BUILD.txt	11 Jun 2003 17:43:47 -0000	1.6
+++ BUILD.txt	14 Jun 2003 02:25:59 -0000
@@ -86,19 +86,15 @@
   http://www.experimentalstuff.com/Technologies/JavaCC/
 
 Follow the download links and download the zip file to a temporary
-location on your file system. Unzip the file and run the large class file
-in the directory. On windows, use this command from the temp directory:
+location on your file system. Unzip the file to the location
+of your choice. (Note that JavaCC 3.0 no longer uses an installer)
 
-  java -cp . JavaCC2_1
-
-This will launch a Java GUI installer.  There is also a command line
-installer available, and the installation class will give you those
-directions.  After JavaCC is installed, edit your build properties
+After JavaCC is installed, edit your build properties
 (as in step 2), and add the line
 
-  javacc.home=/javacc/bin
+  javacc.home=/javacc
 
-where this points to the bin directory of your javacc installation.
+where this points to the directory of your javacc installation.
 
 Step 4) Run ant
 
Index: build.xml
===================================================================
RCS file: /home/cvspublic/jakarta-lucene/build.xml,v
retrieving revision 1.36
diff -u -r1.36 build.xml
--- build.xml	11 Jun 2003 17:43:47 -0000	1.36
+++ build.xml	14 Jun 2003 02:26:00 -0000
@@ -52,10 +52,23 @@
 
     <available
       property="javacc.present"
-      classname="COM.sun.labs.javacc.Main"
-      classpath="${javacc.zip}"
+      classname="org.netbeans.javacc.parser.Main"
+      classpath="${javacc.jar}"
     />
 
+    <!-- ==================================================================
+         Add the JavaCC 3.0 Ant Support by defining the javacc3 Task
+       
+         Note: This is a temporary fix until Ant 1.6 is released
+               then the javacc3 tasks should be changed back to javacc tasks
+         ================================================================== -->
+    <taskdef name="javacc3"
+             classname="org.apache.tools.ant.taskdefs.optional.javacc.JavaCC_3">
+      <classpath>
+        <path refid="classpath"/>
+      </classpath>
+    </taskdef>
+
     <available
       property="junit.present"
       classname="org.apache.tools.ant.taskdefs.optional.junit.JUnitTask"
@@ -72,24 +85,27 @@
       ##################################################################
       JavaCC not found.
       JavaCC Home: ${javacc.home}
-      JavaCC Zip: ${javacc.zip}
+      JavaCC Jar: ${javacc.jar}
 
-      Please download and install JavaCC 2.0 from:
+      Please download and install JavaCC 3.0 from:
 
       &lt;http://www.experimentalstuff.com/Technologies/JavaCC/&gt;
 
+      If you are using a version of Ant prior to 1.6, the javacc.jar
+      in your ${javacc.home}/bin/lib must be renamed to JavaCC.zip
+      
       Then, create a build.properties file either in your home
       directory, or within the Lucene directory and set the javacc.home
-      property to the path where JavaCC.zip is located. For example,
-      if you installed JavaCC in /usr/local/java/javacc2.0, then set the
+      property to the path where JavaCC is located. For example,
+      if you installed JavaCC in /usr/local/java/javacc-3.0, then set the
       javacc.home property to:
 
-      javacc.home=/usr/local/java/javacc2.0/bin
+      javacc.home=/usr/local/java/javacc-3.0
 
       If you get an error like the one below, then you have not installed
       things correctly. Please check all your paths and try again.
 
-      java.lang.NoClassDefFoundError: COM/sun/labs/javacc/Main
+      java.lang.NoClassDefFoundError: org.netbeans.javacc.parser.Main
       ##################################################################
     </echo>
   </target>
@@ -99,25 +115,10 @@
   <!-- ================================================================== -->
   <!--                                                                    -->
   <!-- ================================================================== -->
-  <target name="compile" depends="init,javacc_check" if="javacc.present">
-    <mkdir dir="${build.src}/org/apache/lucene/analysis/standard"/>
-    <javacc
-      target="${src.dir}/org/apache/lucene/analysis/standard/StandardTokenizer.jj"
-      javacchome="${javacc.zip.dir}"
-      outputdirectory="${build.src}/org/apache/lucene/analysis/standard"
-    />
-
-    <delete file="${build.src}/org/apache/lucene/analysis/standard/ParseException.java"/>
-    <mkdir dir="${build.src}/org/apache/lucene/queryParser"/>
-    <javacc
-      target="${src.dir}/org/apache/lucene/queryParser/QueryParser.jj"
-      javacchome="${javacc.zip.dir}"
-      outputdirectory="${build.src}/org/apache/lucene/queryParser"
-    />
-
+  <target name="compile" depends="init">
     <javac
       encoding="${build.encoding}"
-      srcdir="${src.dir}:${build.src}"
+      srcdir="${src.dir}"
       includes="org/**/*.java"
       destdir="${build.classes}"
       debug="${debug}">
@@ -202,22 +203,7 @@
   <!-- ================================================================== -->
   <!--                                                                    -->
   <!-- ================================================================== -->
-  <target name="jar-src" depends="init,javacc_check" if="javacc.present">
-    <mkdir dir="${build.src}/org/apache/lucene/analysis/standard"/>
-    <javacc
-      target="${src.dir}/org/apache/lucene/analysis/standard/StandardTokenizer.jj"
-      javacchome="${javacc.zip.dir}"
-      outputdirectory="${build.src}/org/apache/lucene/analysis/standard"
-    />
-
-    <delete file="${build.src}/org/apache/lucene/analysis/standard/ParseException.java"/>
-    <mkdir dir="${build.src}/org/apache/lucene/queryParser"/>
-    <javacc
-      target="${src.dir}/org/apache/lucene/queryParser/QueryParser.jj"
-      javacchome="${javacc.zip.dir}"
-      outputdirectory="${build.src}/org/apache/lucene/queryParser"
-    />
-
+  <target name="jar-src" depends="init">
     <jar jarfile="${build.dir}/${final.name}-src.jar">
       <fileset dir="${build.dir}" includes="**/*.java"/>
     </jar>
@@ -228,7 +214,7 @@
   <!-- ================================================================== -->
   <!--                                                                    -->
   <!-- ================================================================== -->
-  <target name="demo" depends="compile" if="javacc.present">
+  <target name="demo" depends="compile">
     <mkdir dir="${build.demo}"/>
     <mkdir dir="${build.demo.src}" />
 
@@ -239,11 +225,6 @@
       </fileset>
     </copy>
 
-    <javacc
-      target="${build.demo.src}/org/apache/lucene/demo/html/HTMLParser.jj"
-      javacchome="${javacc.zip.dir}"
-      outputdirectory="${build.demo.src}/org/apache/lucene/demo/html"
-    />
     <mkdir dir="${build.demo.classes}"/>
 
     <javac
@@ -629,4 +610,42 @@
     </war>
   </target>
   -->
+  
+  <!-- ================================================================== -->
+  <!-- Build the JavaCC files into the source tree                                              -->
+  <!-- ================================================================== -->
+  <target name="javacc" depends="javacc-StandardAnalyzer,javacc-QueryParser,javacc-HTMLParser"/>
+  
+  <target name="javacc-StandardAnalyzer" depends="init" if="javacc.present">
+    <!-- generate this in a build directory so we can exclude ParseException -->
+    <mkdir dir="${build.src}/org/apache/lucene/analysis/standard"/>
+    <javacc3
+      target="${src.dir}/org/apache/lucene/analysis/standard/StandardTokenizer.jj"
+      javacchome="${javacc.home}"
+      outputdirectory="${build.src}/org/apache/lucene/analysis/standard"
+    />
+    <copy todir="${src.dir}/org/apache/lucene/analysis/standard">
+      <fileset dir="${build.src}/org/apache/lucene/analysis/standard">
+        <include name="*.java"/>
+        <exclude name="ParseException.java"/>
+      </fileset>
+    </copy>    
+  </target>
+   
+  <target name="javacc-QueryParser" depends="init" if="javacc.present">
+    <javacc3
+      target="${src.dir}/org/apache/lucene/queryParser/QueryParser.jj"
+      javacchome="${javacc.home}"
+      outputdirectory="${src.dir}/org/apache/lucene/queryParser"
+    />
+  </target>
+   
+  <target name="javacc-HTMLParser" depends="init" if="javacc.present">
+    <javacc3
+      target="${demo.src}/org/apache/lucene/demo/html/HTMLParser.jj"
+      javacchome="${javacc.home}"
+      outputdirectory="${demo.src}/org/apache/lucene/demo/html"
+    />
+  </target>
+   
 </project>
Index: default.properties
===================================================================
RCS file: /home/cvspublic/jakarta-lucene/default.properties,v
retrieving revision 1.14
diff -u -r1.14 default.properties
--- default.properties	20 Mar 2003 18:15:04 -0000	1.14
+++ default.properties	14 Jun 2003 02:26:00 -0000
@@ -58,8 +58,8 @@
 
 # Home directory of JavaCC
 javacc.home = .
-javacc.zip.dir = ${javacc.home}/lib
-javacc.zip = ${javacc.zip.dir}/JavaCC.zip
+javacc.lib.dir = ${javacc.home}/bin/lib
+javacc.jar = ${javacc.lib.dir}/javacc.jar
 
 # Home directory of jakarta-site2
 jakarta.site2.home = ../jakarta-site2
Index: lib/javacc3-ant-support.jar
===================================================================
RCS file: lib/javacc3-ant-support.jar
diff -N lib/javacc3-ant-support.jar
Binary files /dev/null and javacc3-ant-support.jar differ
Index: src/demo/org/apache/lucene/demo/html/HTMLParser.java
===================================================================
RCS file: src/demo/org/apache/lucene/demo/html/HTMLParser.java
diff -N src/demo/org/apache/lucene/demo/html/HTMLParser.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/demo/org/apache/lucene/demo/html/HTMLParser.java	14 Jun 2003 02:26:01 -0000
@@ -0,0 +1,688 @@
+/* Generated By:JavaCC: Do not edit this line. HTMLParser.java */
+package org.apache.lucene.demo.html;
+
+import java.io.*;
+import java.util.Properties;
+
+public class HTMLParser implements HTMLParserConstants {
+  public static int SUMMARY_LENGTH = 200;
+
+  StringBuffer title = new StringBuffer(SUMMARY_LENGTH);
+  StringBuffer summary = new StringBuffer(SUMMARY_LENGTH * 2);
+  Properties metaTags=new Properties();
+  String currentMetaTag="";
+  int length = 0;
+  boolean titleComplete = false;
+  boolean inTitle = false;
+  boolean inMetaTag = false;
+  boolean inStyle = false;
+  boolean inScript = false;
+  boolean afterTag = false;
+  boolean afterSpace = false;
+  String eol = System.getProperty("line.separator");
+  PipedReader pipeIn = null;
+  PipedWriter pipeOut;
+
+  public HTMLParser(File file) throws FileNotFoundException {
+    this(new FileInputStream(file));
+  }
+
+  public String getTitle() throws IOException, InterruptedException {
+    if (pipeIn == null)
+      getReader();                                // spawn parsing thread
+    while (true) {
+      synchronized(this) {
+        if (titleComplete || (length > SUMMARY_LENGTH))
+          break;
+        wait(10);
+      }
+    }
+    return title.toString().trim();
+  }
+
+  public Properties getMetaTags() throws IOException,
+InterruptedException {
+    if (pipeIn == null)
+      getReader();                                // spawn parsing thread
+    while (true) {
+      synchronized(this) {
+        if (titleComplete || (length > SUMMARY_LENGTH))
+          break;
+        wait(10);
+      }
+    }
+    return metaTags;
+  }
+
+
+  public String getSummary() throws IOException, InterruptedException {
+    if (pipeIn == null)
+      getReader();                                // spawn parsing thread
+    while (true) {
+      synchronized(this) {
+        if (summary.length() >= SUMMARY_LENGTH)
+          break;
+        wait(10);
+      }
+    }
+    if (summary.length() > SUMMARY_LENGTH)
+      summary.setLength(SUMMARY_LENGTH);
+
+    String sum = summary.toString().trim();
+    String tit = getTitle();
+    if (sum.startsWith(tit))
+      return sum.substring(tit.length());
+    else
+      return sum;
+  }
+
+  public Reader getReader() throws IOException {
+    if (pipeIn == null) {
+      pipeIn = new PipedReader();
+      pipeOut = new PipedWriter(pipeIn);
+
+      Thread thread = new ParserThread(this);
+      thread.start();                             // start parsing
+    }
+
+    return pipeIn;
+  }
+
+  void addToSummary(String text) {
+    if (summary.length() < SUMMARY_LENGTH) {
+      summary.append(text);
+      if (summary.length() >= SUMMARY_LENGTH) {
+        synchronized(this) {
+          notifyAll();
+        }
+      }
+    }
+  }
+
+  void addText(String text) throws IOException {
+    if (inScript)
+      return;
+    if (inStyle)
+      return;
+    if (inMetaTag)
+    {
+        metaTags.setProperty(currentMetaTag, text);
+        return;
+    }
+    if (inTitle)
+      title.append(text);
+    else {
+      addToSummary(text);
+      if (!titleComplete && !title.equals("")) {  // finished title
+        synchronized(this) {
+          titleComplete = true;                   // tell waiting threads
+          notifyAll();
+        }
+      }
+    }
+
+    length += text.length();
+    pipeOut.write(text);
+
+    afterSpace = false;
+  }
+
+  void addSpace() throws IOException {
+    if (inScript)
+      return;
+    if (!afterSpace) {
+      if (inTitle)
+        title.append(" ");
+      else
+        addToSummary(" ");
+
+      String space = afterTag ? eol : " ";
+      length += space.length();
+      pipeOut.write(space);
+      afterSpace = true;
+    }
+  }
+
+  final public void HTMLDocument() throws ParseException, IOException {
+  Token t;
+    label_1:
+    while (true) {
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case TagName:
+      case DeclName:
+      case Comment1:
+      case Comment2:
+      case Word:
+      case Entity:
+      case Space:
+      case Punct:
+        ;
+        break;
+      default:
+        jj_la1[0] = jj_gen;
+        break label_1;
+      }
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case TagName:
+        Tag();
+                      afterTag = true;
+        break;
+      case DeclName:
+        t = Decl();
+                      afterTag = true;
+        break;
+      case Comment1:
+      case Comment2:
+        CommentTag();
+                      afterTag = true;
+        break;
+      case Word:
+        t = jj_consume_token(Word);
+                      addText(t.image); afterTag = false;
+        break;
+      case Entity:
+        t = jj_consume_token(Entity);
+                      addText(Entities.decode(t.image)); afterTag = false;
+        break;
+      case Punct:
+        t = jj_consume_token(Punct);
+                      addText(t.image); afterTag = false;
+        break;
+      case Space:
+        jj_consume_token(Space);
+                      addSpace(); afterTag = false;
+        break;
+      default:
+        jj_la1[1] = jj_gen;
+        jj_consume_token(-1);
+        throw new ParseException();
+      }
+    }
+    jj_consume_token(0);
+  }
+
+  final public void Tag() throws ParseException, IOException {
+  Token t1, t2;
+  boolean inImg = false;
+    t1 = jj_consume_token(TagName);
+    inTitle = t1.image.equalsIgnoreCase("<title"); // keep track if in <TITLE>
+    inMetaTag = t1.image.equalsIgnoreCase("<META"); // keep track if in <META>
+    inStyle = t1.image.equalsIgnoreCase("<STYLE"); // keep track if in <STYLE>
+    inImg = t1.image.equalsIgnoreCase("<img");    // keep track if in <IMG>
+    if (inScript) {                               // keep track if in <SCRIPT>
+      inScript = !t1.image.equalsIgnoreCase("</script");
+    } else {
+      inScript = t1.image.equalsIgnoreCase("<script");
+    }
+    label_2:
+    while (true) {
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case ArgName:
+        ;
+        break;
+      default:
+        jj_la1[2] = jj_gen;
+        break label_2;
+      }
+      t1 = jj_consume_token(ArgName);
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case ArgEquals:
+        jj_consume_token(ArgEquals);
+        switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+        case ArgValue:
+        case ArgQuote1:
+        case ArgQuote2:
+          t2 = ArgValue();
+       if (inImg && t1.image.equalsIgnoreCase("alt") && t2 != null)
+         addText("[" + t2.image + "]");
+
+        if(inMetaTag &&
+                        (  t1.image.equalsIgnoreCase("name") ||
+                           t1.image.equalsIgnoreCase("HTTP-EQUIV")
+                        )
+           && t2 != null)
+        {
+                currentMetaTag=t2.image.toLowerCase();
+        }
+        if(inMetaTag && t1.image.equalsIgnoreCase("content") && t2 !=
+null)
+        {
+                addText(t2.image);
+        }
+          break;
+        default:
+          jj_la1[3] = jj_gen;
+          ;
+        }
+        break;
+      default:
+        jj_la1[4] = jj_gen;
+        ;
+      }
+    }
+    jj_consume_token(TagEnd);
+  }
+
+  final public Token ArgValue() throws ParseException {
+  Token t = null;
+    switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+    case ArgValue:
+      t = jj_consume_token(ArgValue);
+                                              {if (true) return t;}
+      break;
+    default:
+      jj_la1[5] = jj_gen;
+      if (jj_2_1(2)) {
+        jj_consume_token(ArgQuote1);
+        jj_consume_token(CloseQuote1);
+                                              {if (true) return t;}
+      } else {
+        switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+        case ArgQuote1:
+          jj_consume_token(ArgQuote1);
+          t = jj_consume_token(Quote1Text);
+          jj_consume_token(CloseQuote1);
+                                              {if (true) return t;}
+          break;
+        default:
+          jj_la1[6] = jj_gen;
+          if (jj_2_2(2)) {
+            jj_consume_token(ArgQuote2);
+            jj_consume_token(CloseQuote2);
+                                              {if (true) return t;}
+          } else {
+            switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+            case ArgQuote2:
+              jj_consume_token(ArgQuote2);
+              t = jj_consume_token(Quote2Text);
+              jj_consume_token(CloseQuote2);
+                                              {if (true) return t;}
+              break;
+            default:
+              jj_la1[7] = jj_gen;
+              jj_consume_token(-1);
+              throw new ParseException();
+            }
+          }
+        }
+      }
+    }
+    throw new Error("Missing return statement in function");
+  }
+
+  final public Token Decl() throws ParseException {
+  Token t;
+    t = jj_consume_token(DeclName);
+    label_3:
+    while (true) {
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case ArgName:
+      case ArgEquals:
+      case ArgValue:
+      case ArgQuote1:
+      case ArgQuote2:
+        ;
+        break;
+      default:
+        jj_la1[8] = jj_gen;
+        break label_3;
+      }
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case ArgName:
+        jj_consume_token(ArgName);
+        break;
+      case ArgValue:
+      case ArgQuote1:
+      case ArgQuote2:
+        ArgValue();
+        break;
+      case ArgEquals:
+        jj_consume_token(ArgEquals);
+        break;
+      default:
+        jj_la1[9] = jj_gen;
+        jj_consume_token(-1);
+        throw new ParseException();
+      }
+    }
+    jj_consume_token(TagEnd);
+    {if (true) return t;}
+    throw new Error("Missing return statement in function");
+  }
+
+  final public void CommentTag() throws ParseException {
+    switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+    case Comment1:
+      jj_consume_token(Comment1);
+      label_4:
+      while (true) {
+        switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+        case CommentText1:
+          ;
+          break;
+        default:
+          jj_la1[10] = jj_gen;
+          break label_4;
+        }
+        jj_consume_token(CommentText1);
+      }
+      jj_consume_token(CommentEnd1);
+      break;
+    case Comment2:
+      jj_consume_token(Comment2);
+      label_5:
+      while (true) {
+        switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+        case CommentText2:
+          ;
+          break;
+        default:
+          jj_la1[11] = jj_gen;
+          break label_5;
+        }
+        jj_consume_token(CommentText2);
+      }
+      jj_consume_token(CommentEnd2);
+      break;
+    default:
+      jj_la1[12] = jj_gen;
+      jj_consume_token(-1);
+      throw new ParseException();
+    }
+  }
+
+  final private boolean jj_2_1(int xla) {
+    jj_la = xla; jj_lastpos = jj_scanpos = token;
+    boolean retval = !jj_3_1();
+    jj_save(0, xla);
+    return retval;
+  }
+
+  final private boolean jj_2_2(int xla) {
+    jj_la = xla; jj_lastpos = jj_scanpos = token;
+    boolean retval = !jj_3_2();
+    jj_save(1, xla);
+    return retval;
+  }
+
+  final private boolean jj_3_1() {
+    if (jj_scan_token(ArgQuote1)) return true;
+    if (jj_la == 0 && jj_scanpos == jj_lastpos) return false;
+    if (jj_scan_token(CloseQuote1)) return true;
+    if (jj_la == 0 && jj_scanpos == jj_lastpos) return false;
+    return false;
+  }
+
+  final private boolean jj_3_2() {
+    if (jj_scan_token(ArgQuote2)) return true;
+    if (jj_la == 0 && jj_scanpos == jj_lastpos) return false;
+    if (jj_scan_token(CloseQuote2)) return true;
+    if (jj_la == 0 && jj_scanpos == jj_lastpos) return false;
+    return false;
+  }
+
+  public HTMLParserTokenManager token_source;
+  SimpleCharStream jj_input_stream;
+  public Token token, jj_nt;
+  private int jj_ntk;
+  private Token jj_scanpos, jj_lastpos;
+  private int jj_la;
+  public boolean lookingAhead = false;
+  private boolean jj_semLA;
+  private int jj_gen;
+  final private int[] jj_la1 = new int[13];
+  static private int[] jj_la1_0;
+  static {
+      jj_la1_0();
+   }
+   private static void jj_la1_0() {
+      jj_la1_0 = new int[] {0xb3e,0xb3e,0x1000,0x38000,0x2000,0x8000,0x10000,0x20000,0x3b000,0x3b000,0x800000,0x2000000,0x18,};
+   }
+  final private JJCalls[] jj_2_rtns = new JJCalls[2];
+  private boolean jj_rescan = false;
+  private int jj_gc = 0;
+
+  public HTMLParser(java.io.InputStream stream) {
+    jj_input_stream = new SimpleCharStream(stream, 1, 1);
+    token_source = new HTMLParserTokenManager(jj_input_stream);
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 13; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  public void ReInit(java.io.InputStream stream) {
+    jj_input_stream.ReInit(stream, 1, 1);
+    token_source.ReInit(jj_input_stream);
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 13; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  public HTMLParser(java.io.Reader stream) {
+    jj_input_stream = new SimpleCharStream(stream, 1, 1);
+    token_source = new HTMLParserTokenManager(jj_input_stream);
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 13; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  public void ReInit(java.io.Reader stream) {
+    jj_input_stream.ReInit(stream, 1, 1);
+    token_source.ReInit(jj_input_stream);
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 13; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  public HTMLParser(HTMLParserTokenManager tm) {
+    token_source = tm;
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 13; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  public void ReInit(HTMLParserTokenManager tm) {
+    token_source = tm;
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 13; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  final private Token jj_consume_token(int kind) throws ParseException {
+    Token oldToken;
+    if ((oldToken = token).next != null) token = token.next;
+    else token = token.next = token_source.getNextToken();
+    jj_ntk = -1;
+    if (token.kind == kind) {
+      jj_gen++;
+      if (++jj_gc > 100) {
+        jj_gc = 0;
+        for (int i = 0; i < jj_2_rtns.length; i++) {
+          JJCalls c = jj_2_rtns[i];
+          while (c != null) {
+            if (c.gen < jj_gen) c.first = null;
+            c = c.next;
+          }
+        }
+      }
+      return token;
+    }
+    token = oldToken;
+    jj_kind = kind;
+    throw generateParseException();
+  }
+
+  final private boolean jj_scan_token(int kind) {
+    if (jj_scanpos == jj_lastpos) {
+      jj_la--;
+      if (jj_scanpos.next == null) {
+        jj_lastpos = jj_scanpos = jj_scanpos.next = token_source.getNextToken();
+      } else {
+        jj_lastpos = jj_scanpos = jj_scanpos.next;
+      }
+    } else {
+      jj_scanpos = jj_scanpos.next;
+    }
+    if (jj_rescan) {
+      int i = 0; Token tok = token;
+      while (tok != null && tok != jj_scanpos) { i++; tok = tok.next; }
+      if (tok != null) jj_add_error_token(kind, i);
+    }
+    return (jj_scanpos.kind != kind);
+  }
+
+  final public Token getNextToken() {
+    if (token.next != null) token = token.next;
+    else token = token.next = token_source.getNextToken();
+    jj_ntk = -1;
+    jj_gen++;
+    return token;
+  }
+
+  final public Token getToken(int index) {
+    Token t = lookingAhead ? jj_scanpos : token;
+    for (int i = 0; i < index; i++) {
+      if (t.next != null) t = t.next;
+      else t = t.next = token_source.getNextToken();
+    }
+    return t;
+  }
+
+  final private int jj_ntk() {
+    if ((jj_nt=token.next) == null)
+      return (jj_ntk = (token.next=token_source.getNextToken()).kind);
+    else
+      return (jj_ntk = jj_nt.kind);
+  }
+
+  private java.util.Vector jj_expentries = new java.util.Vector();
+  private int[] jj_expentry;
+  private int jj_kind = -1;
+  private int[] jj_lasttokens = new int[100];
+  private int jj_endpos;
+
+  private void jj_add_error_token(int kind, int pos) {
+    if (pos >= 100) return;
+    if (pos == jj_endpos + 1) {
+      jj_lasttokens[jj_endpos++] = kind;
+    } else if (jj_endpos != 0) {
+      jj_expentry = new int[jj_endpos];
+      for (int i = 0; i < jj_endpos; i++) {
+        jj_expentry[i] = jj_lasttokens[i];
+      }
+      boolean exists = false;
+      for (java.util.Enumeration enum = jj_expentries.elements(); enum.hasMoreElements();) {
+        int[] oldentry = (int[])(enum.nextElement());
+        if (oldentry.length == jj_expentry.length) {
+          exists = true;
+          for (int i = 0; i < jj_expentry.length; i++) {
+            if (oldentry[i] != jj_expentry[i]) {
+              exists = false;
+              break;
+            }
+          }
+          if (exists) break;
+        }
+      }
+      if (!exists) jj_expentries.addElement(jj_expentry);
+      if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind;
+    }
+  }
+
+  public ParseException generateParseException() {
+    jj_expentries.removeAllElements();
+    boolean[] la1tokens = new boolean[27];
+    for (int i = 0; i < 27; i++) {
+      la1tokens[i] = false;
+    }
+    if (jj_kind >= 0) {
+      la1tokens[jj_kind] = true;
+      jj_kind = -1;
+    }
+    for (int i = 0; i < 13; i++) {
+      if (jj_la1[i] == jj_gen) {
+        for (int j = 0; j < 32; j++) {
+          if ((jj_la1_0[i] & (1<<j)) != 0) {
+            la1tokens[j] = true;
+          }
+        }
+      }
+    }
+    for (int i = 0; i < 27; i++) {
+      if (la1tokens[i]) {
+        jj_expentry = new int[1];
+        jj_expentry[0] = i;
+        jj_expentries.addElement(jj_expentry);
+      }
+    }
+    jj_endpos = 0;
+    jj_rescan_token();
+    jj_add_error_token(0, 0);
+    int[][] exptokseq = new int[jj_expentries.size()][];
+    for (int i = 0; i < jj_expentries.size(); i++) {
+      exptokseq[i] = (int[])jj_expentries.elementAt(i);
+    }
+    return new ParseException(token, exptokseq, tokenImage);
+  }
+
+  final public void enable_tracing() {
+  }
+
+  final public void disable_tracing() {
+  }
+
+  final private void jj_rescan_token() {
+    jj_rescan = true;
+    for (int i = 0; i < 2; i++) {
+      JJCalls p = jj_2_rtns[i];
+      do {
+        if (p.gen > jj_gen) {
+          jj_la = p.arg; jj_lastpos = jj_scanpos = p.first;
+          switch (i) {
+            case 0: jj_3_1(); break;
+            case 1: jj_3_2(); break;
+          }
+        }
+        p = p.next;
+      } while (p != null);
+    }
+    jj_rescan = false;
+  }
+
+  final private void jj_save(int index, int xla) {
+    JJCalls p = jj_2_rtns[index];
+    while (p.gen > jj_gen) {
+      if (p.next == null) { p = p.next = new JJCalls(); break; }
+      p = p.next;
+    }
+    p.gen = jj_gen + xla - jj_la; p.first = token; p.arg = xla;
+  }
+
+  static final class JJCalls {
+    int gen;
+    Token first;
+    int arg;
+    JJCalls next;
+  }
+
+//    void handleException(Exception e) {
+//      System.out.println(e.toString());  // print the error message
+//      System.out.println("Skipping...");
+//      Token t;
+//      do {
+//        t = getNextToken();
+//      } while (t.kind != TagEnd);
+//    }
+}
Index: src/demo/org/apache/lucene/demo/html/HTMLParserConstants.java
===================================================================
RCS file: src/demo/org/apache/lucene/demo/html/HTMLParserConstants.java
diff -N src/demo/org/apache/lucene/demo/html/HTMLParserConstants.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/demo/org/apache/lucene/demo/html/HTMLParserConstants.java	14 Jun 2003 02:26:01 -0000
@@ -0,0 +1,71 @@
+/* Generated By:JavaCC: Do not edit this line. HTMLParserConstants.java */
+package org.apache.lucene.demo.html;
+
+public interface HTMLParserConstants {
+
+  int EOF = 0;
+  int TagName = 1;
+  int DeclName = 2;
+  int Comment1 = 3;
+  int Comment2 = 4;
+  int Word = 5;
+  int LET = 6;
+  int NUM = 7;
+  int Entity = 8;
+  int Space = 9;
+  int SP = 10;
+  int Punct = 11;
+  int ArgName = 12;
+  int ArgEquals = 13;
+  int TagEnd = 14;
+  int ArgValue = 15;
+  int ArgQuote1 = 16;
+  int ArgQuote2 = 17;
+  int Quote1Text = 19;
+  int CloseQuote1 = 20;
+  int Quote2Text = 21;
+  int CloseQuote2 = 22;
+  int CommentText1 = 23;
+  int CommentEnd1 = 24;
+  int CommentText2 = 25;
+  int CommentEnd2 = 26;
+
+  int DEFAULT = 0;
+  int WithinTag = 1;
+  int AfterEquals = 2;
+  int WithinQuote1 = 3;
+  int WithinQuote2 = 4;
+  int WithinComment1 = 5;
+  int WithinComment2 = 6;
+
+  String[] tokenImage = {
+    "<EOF>",
+    "<TagName>",
+    "<DeclName>",
+    "\"<!--\"",
+    "\"<!\"",
+    "<Word>",
+    "<LET>",
+    "<NUM>",
+    "<Entity>",
+    "<Space>",
+    "<SP>",
+    "<Punct>",
+    "<ArgName>",
+    "\"=\"",
+    "<TagEnd>",
+    "<ArgValue>",
+    "\"\\\'\"",
+    "\"\\\"\"",
+    "<token of kind 18>",
+    "<Quote1Text>",
+    "<CloseQuote1>",
+    "<Quote2Text>",
+    "<CloseQuote2>",
+    "<CommentText1>",
+    "\"-->\"",
+    "<CommentText2>",
+    "\">\"",
+  };
+
+}
Index: src/demo/org/apache/lucene/demo/html/HTMLParserTokenManager.java
===================================================================
RCS file: src/demo/org/apache/lucene/demo/html/HTMLParserTokenManager.java
diff -N src/demo/org/apache/lucene/demo/html/HTMLParserTokenManager.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/demo/org/apache/lucene/demo/html/HTMLParserTokenManager.java	14 Jun 2003 02:26:01 -0000
@@ -0,0 +1,1350 @@
+/* Generated By:JavaCC: Do not edit this line. HTMLParserTokenManager.java */
+package org.apache.lucene.demo.html;
+import java.io.*;
+import java.util.Properties;
+
+public class HTMLParserTokenManager implements HTMLParserConstants
+{
+  public  java.io.PrintStream debugStream = System.out;
+  public  void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
+private final int jjStopStringLiteralDfa_0(int pos, long active0)
+{
+   switch (pos)
+   {
+      case 0:
+         if ((active0 & 0x18L) != 0L)
+            return 17;
+         return -1;
+      case 1:
+         if ((active0 & 0x18L) != 0L)
+            return 22;
+         return -1;
+      default :
+         return -1;
+   }
+}
+private final int jjStartNfa_0(int pos, long active0)
+{
+   return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0), pos + 1);
+}
+private final int jjStopAtPos(int pos, int kind)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   return pos + 1;
+}
+private final int jjStartNfaWithStates_0(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_0(state, pos + 1);
+}
+private final int jjMoveStringLiteralDfa0_0()
+{
+   switch(curChar)
+   {
+      case 60:
+         return jjMoveStringLiteralDfa1_0(0x18L);
+      default :
+         return jjMoveNfa_0(11, 0);
+   }
+}
+private final int jjMoveStringLiteralDfa1_0(long active0)
+{
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) {
+      jjStopStringLiteralDfa_0(0, active0);
+      return 1;
+   }
+   switch(curChar)
+   {
+      case 33:
+         if ((active0 & 0x10L) != 0L)
+         {
+            jjmatchedKind = 4;
+            jjmatchedPos = 1;
+         }
+         return jjMoveStringLiteralDfa2_0(active0, 0x8L);
+      default :
+         break;
+   }
+   return jjStartNfa_0(0, active0);
+}
+private final int jjMoveStringLiteralDfa2_0(long old0, long active0)
+{
+   if (((active0 &= old0)) == 0L)
+      return jjStartNfa_0(0, old0); 
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) {
+      jjStopStringLiteralDfa_0(1, active0);
+      return 2;
+   }
+   switch(curChar)
+   {
+      case 45:
+         return jjMoveStringLiteralDfa3_0(active0, 0x8L);
+      default :
+         break;
+   }
+   return jjStartNfa_0(1, active0);
+}
+private final int jjMoveStringLiteralDfa3_0(long old0, long active0)
+{
+   if (((active0 &= old0)) == 0L)
+      return jjStartNfa_0(1, old0); 
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) {
+      jjStopStringLiteralDfa_0(2, active0);
+      return 3;
+   }
+   switch(curChar)
+   {
+      case 45:
+         if ((active0 & 0x8L) != 0L)
+            return jjStopAtPos(3, 3);
+         break;
+      default :
+         break;
+   }
+   return jjStartNfa_0(2, active0);
+}
+private final void jjCheckNAdd(int state)
+{
+   if (jjrounds[state] != jjround)
+   {
+      jjstateSet[jjnewStateCnt++] = state;
+      jjrounds[state] = jjround;
+   }
+}
+private final void jjAddStates(int start, int end)
+{
+   do {
+      jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+   } while (start++ != end);
+}
+private final void jjCheckNAddTwoStates(int state1, int state2)
+{
+   jjCheckNAdd(state1);
+   jjCheckNAdd(state2);
+}
+private final void jjCheckNAddStates(int start, int end)
+{
+   do {
+      jjCheckNAdd(jjnextStates[start]);
+   } while (start++ != end);
+}
+private final void jjCheckNAddStates(int start)
+{
+   jjCheckNAdd(jjnextStates[start]);
+   jjCheckNAdd(jjnextStates[start + 1]);
+}
+static final long[] jjbitVec0 = {
+   0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+private final int jjMoveNfa_0(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 25;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 11:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(7, 2);
+                  else if ((0x100002600L & l) != 0L)
+                  {
+                     if (kind > 9)
+                        kind = 9;
+                     jjCheckNAdd(10);
+                  }
+                  else if (curChar == 60)
+                     jjCheckNAddStates(0, 2);
+                  else if (curChar == 38)
+                     jjAddStates(3, 4);
+                  else if (curChar == 36)
+                     jjstateSet[jjnewStateCnt++] = 1;
+                  if ((0x3ff000000000000L & l) != 0L)
+                  {
+                     if (kind > 5)
+                        kind = 5;
+                     jjCheckNAddStates(5, 9);
+                  }
+                  break;
+               case 17:
+                  if (curChar == 33)
+                     jjstateSet[jjnewStateCnt++] = 22;
+                  else if (curChar == 47)
+                     jjCheckNAdd(18);
+                  break;
+               case 0:
+                  if (curChar == 36)
+                     jjstateSet[jjnewStateCnt++] = 1;
+                  break;
+               case 1:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAdd(2);
+                  break;
+               case 2:
+                  if ((0x500000000000L & l) != 0L)
+                     jjstateSet[jjnewStateCnt++] = 3;
+                  break;
+               case 3:
+               case 9:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddStates(10, 12);
+                  break;
+               case 4:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddStates(5, 9);
+                  break;
+               case 5:
+                  if ((0x880000000000L & l) == 0L)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddStates(13, 16);
+                  break;
+               case 6:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(7, 2);
+                  break;
+               case 7:
+                  if (curChar != 34)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddStates(10, 12);
+                  break;
+               case 8:
+                  if ((0x208000000000L & l) != 0L)
+                     jjstateSet[jjnewStateCnt++] = 9;
+                  break;
+               case 10:
+                  if ((0x100002600L & l) == 0L)
+                     break;
+                  kind = 9;
+                  jjCheckNAdd(10);
+                  break;
+               case 13:
+                  if (curChar == 59 && kind > 8)
+                     kind = 8;
+                  break;
+               case 14:
+                  if (curChar == 35)
+                     jjCheckNAdd(15);
+                  break;
+               case 15:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 8)
+                     kind = 8;
+                  jjCheckNAddTwoStates(15, 13);
+                  break;
+               case 16:
+                  if (curChar == 60)
+                     jjCheckNAddStates(0, 2);
+                  break;
+               case 19:
+                  if ((0x9fffff7affffd9ffL & l) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAdd(20);
+                  break;
+               case 20:
+                  if ((0x9ffffffeffffd9ffL & l) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAdd(20);
+                  break;
+               case 21:
+                  if (curChar == 33)
+                     jjstateSet[jjnewStateCnt++] = 22;
+                  break;
+               case 23:
+                  if ((0x9fffff7affffd9ffL & l) == 0L)
+                     break;
+                  if (kind > 2)
+                     kind = 2;
+                  jjCheckNAdd(24);
+                  break;
+               case 24:
+                  if ((0x9ffffffeffffd9ffL & l) == 0L)
+                     break;
+                  if (kind > 2)
+                     kind = 2;
+                  jjCheckNAdd(24);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 11:
+               case 4:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddStates(5, 9);
+                  break;
+               case 17:
+               case 18:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjstateSet[jjnewStateCnt++] = 19;
+                  break;
+               case 9:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddStates(10, 12);
+                  break;
+               case 12:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 8)
+                     kind = 8;
+                  jjAddStates(17, 18);
+                  break;
+               case 19:
+               case 20:
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAdd(20);
+                  break;
+               case 22:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 2)
+                     kind = 2;
+                  jjstateSet[jjnewStateCnt++] = 23;
+                  break;
+               case 23:
+               case 24:
+                  if (kind > 2)
+                     kind = 2;
+                  jjCheckNAdd(24);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 19:
+               case 20:
+                  if ((jjbitVec0[i2] & l2) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAdd(20);
+                  break;
+               case 23:
+               case 24:
+                  if ((jjbitVec0[i2] & l2) == 0L)
+                     break;
+                  if (kind > 2)
+                     kind = 2;
+                  jjCheckNAdd(24);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 25 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+private final int jjMoveStringLiteralDfa0_4()
+{
+   return jjMoveNfa_4(1, 0);
+}
+private final int jjMoveNfa_4(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 2;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 1:
+                  if ((0xfffffffbffffffffL & l) != 0L)
+                  {
+                     if (kind > 21)
+                        kind = 21;
+                     jjCheckNAdd(0);
+                  }
+                  else if (curChar == 34)
+                  {
+                     if (kind > 22)
+                        kind = 22;
+                  }
+                  break;
+               case 0:
+                  if ((0xfffffffbffffffffL & l) == 0L)
+                     break;
+                  kind = 21;
+                  jjCheckNAdd(0);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 1:
+               case 0:
+                  kind = 21;
+                  jjCheckNAdd(0);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 1:
+               case 0:
+                  if ((jjbitVec0[i2] & l2) == 0L)
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAdd(0);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 2 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+private final int jjStopStringLiteralDfa_6(int pos, long active0)
+{
+   switch (pos)
+   {
+      default :
+         return -1;
+   }
+}
+private final int jjStartNfa_6(int pos, long active0)
+{
+   return jjMoveNfa_6(jjStopStringLiteralDfa_6(pos, active0), pos + 1);
+}
+private final int jjStartNfaWithStates_6(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_6(state, pos + 1);
+}
+private final int jjMoveStringLiteralDfa0_6()
+{
+   switch(curChar)
+   {
+      case 62:
+         return jjStopAtPos(0, 26);
+      default :
+         return jjMoveNfa_6(0, 0);
+   }
+}
+private final int jjMoveNfa_6(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 1;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0xbfffffffffffffffL & l) == 0L)
+                     break;
+                  kind = 25;
+                  jjstateSet[jjnewStateCnt++] = 0;
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  kind = 25;
+                  jjstateSet[jjnewStateCnt++] = 0;
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((jjbitVec0[i2] & l2) == 0L)
+                     break;
+                  if (kind > 25)
+                     kind = 25;
+                  jjstateSet[jjnewStateCnt++] = 0;
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 1 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+private final int jjMoveStringLiteralDfa0_3()
+{
+   return jjMoveNfa_3(1, 0);
+}
+private final int jjMoveNfa_3(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 2;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 1:
+                  if ((0xffffff7fffffffffL & l) != 0L)
+                  {
+                     if (kind > 19)
+                        kind = 19;
+                     jjCheckNAdd(0);
+                  }
+                  else if (curChar == 39)
+                  {
+                     if (kind > 20)
+                        kind = 20;
+                  }
+                  break;
+               case 0:
+                  if ((0xffffff7fffffffffL & l) == 0L)
+                     break;
+                  kind = 19;
+                  jjCheckNAdd(0);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 1:
+               case 0:
+                  kind = 19;
+                  jjCheckNAdd(0);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 1:
+               case 0:
+                  if ((jjbitVec0[i2] & l2) == 0L)
+                     break;
+                  if (kind > 19)
+                     kind = 19;
+                  jjCheckNAdd(0);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 2 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+private final int jjStopStringLiteralDfa_2(int pos, long active0)
+{
+   switch (pos)
+   {
+      default :
+         return -1;
+   }
+}
+private final int jjStartNfa_2(int pos, long active0)
+{
+   return jjMoveNfa_2(jjStopStringLiteralDfa_2(pos, active0), pos + 1);
+}
+private final int jjStartNfaWithStates_2(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_2(state, pos + 1);
+}
+private final int jjMoveStringLiteralDfa0_2()
+{
+   switch(curChar)
+   {
+      case 34:
+         return jjStopAtPos(0, 17);
+      case 39:
+         return jjStopAtPos(0, 16);
+      default :
+         return jjMoveNfa_2(0, 0);
+   }
+}
+private final int jjMoveNfa_2(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 3;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0x9fffff7affffd9ffL & l) != 0L)
+                  {
+                     if (kind > 15)
+                        kind = 15;
+                     jjCheckNAdd(1);
+                  }
+                  else if ((0x100002600L & l) != 0L)
+                  {
+                     if (kind > 18)
+                        kind = 18;
+                     jjCheckNAdd(2);
+                  }
+                  break;
+               case 1:
+                  if ((0xbffffffeffffd9ffL & l) == 0L)
+                     break;
+                  if (kind > 15)
+                     kind = 15;
+                  jjCheckNAdd(1);
+                  break;
+               case 2:
+                  if ((0x100002600L & l) == 0L)
+                     break;
+                  kind = 18;
+                  jjCheckNAdd(2);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+               case 1:
+                  if (kind > 15)
+                     kind = 15;
+                  jjCheckNAdd(1);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+               case 1:
+                  if ((jjbitVec0[i2] & l2) == 0L)
+                     break;
+                  if (kind > 15)
+                     kind = 15;
+                  jjCheckNAdd(1);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 3 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+private final int jjStopStringLiteralDfa_5(int pos, long active0)
+{
+   switch (pos)
+   {
+      case 0:
+         if ((active0 & 0x1000000L) != 0L)
+         {
+            jjmatchedKind = 23;
+            return -1;
+         }
+         return -1;
+      case 1:
+         if ((active0 & 0x1000000L) != 0L)
+         {
+            if (jjmatchedPos == 0)
+            {
+               jjmatchedKind = 23;
+               jjmatchedPos = 0;
+            }
+            return -1;
+         }
+         return -1;
+      default :
+         return -1;
+   }
+}
+private final int jjStartNfa_5(int pos, long active0)
+{
+   return jjMoveNfa_5(jjStopStringLiteralDfa_5(pos, active0), pos + 1);
+}
+private final int jjStartNfaWithStates_5(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_5(state, pos + 1);
+}
+private final int jjMoveStringLiteralDfa0_5()
+{
+   switch(curChar)
+   {
+      case 45:
+         return jjMoveStringLiteralDfa1_5(0x1000000L);
+      default :
+         return jjMoveNfa_5(1, 0);
+   }
+}
+private final int jjMoveStringLiteralDfa1_5(long active0)
+{
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) {
+      jjStopStringLiteralDfa_5(0, active0);
+      return 1;
+   }
+   switch(curChar)
+   {
+      case 45:
+         return jjMoveStringLiteralDfa2_5(active0, 0x1000000L);
+      default :
+         break;
+   }
+   return jjStartNfa_5(0, active0);
+}
+private final int jjMoveStringLiteralDfa2_5(long old0, long active0)
+{
+   if (((active0 &= old0)) == 0L)
+      return jjStartNfa_5(0, old0); 
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) {
+      jjStopStringLiteralDfa_5(1, active0);
+      return 2;
+   }
+   switch(curChar)
+   {
+      case 62:
+         if ((active0 & 0x1000000L) != 0L)
+            return jjStopAtPos(2, 24);
+         break;
+      default :
+         break;
+   }
+   return jjStartNfa_5(1, active0);
+}
+private final int jjMoveNfa_5(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 2;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 1:
+                  if ((0xffffdfffffffffffL & l) != 0L)
+                  {
+                     if (kind > 23)
+                        kind = 23;
+                     jjCheckNAdd(0);
+                  }
+                  else if (curChar == 45)
+                  {
+                     if (kind > 23)
+                        kind = 23;
+                  }
+                  break;
+               case 0:
+                  if ((0xffffdfffffffffffL & l) == 0L)
+                     break;
+                  kind = 23;
+                  jjCheckNAdd(0);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 1:
+               case 0:
+                  kind = 23;
+                  jjCheckNAdd(0);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 1:
+               case 0:
+                  if ((jjbitVec0[i2] & l2) == 0L)
+                     break;
+                  if (kind > 23)
+                     kind = 23;
+                  jjCheckNAdd(0);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 2 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+private final int jjStopStringLiteralDfa_1(int pos, long active0)
+{
+   switch (pos)
+   {
+      default :
+         return -1;
+   }
+}
+private final int jjStartNfa_1(int pos, long active0)
+{
+   return jjMoveNfa_1(jjStopStringLiteralDfa_1(pos, active0), pos + 1);
+}
+private final int jjStartNfaWithStates_1(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_1(state, pos + 1);
+}
+private final int jjMoveStringLiteralDfa0_1()
+{
+   switch(curChar)
+   {
+      case 34:
+         return jjStopAtPos(0, 17);
+      case 39:
+         return jjStopAtPos(0, 16);
+      case 61:
+         return jjStartNfaWithStates_1(0, 13, 3);
+      default :
+         return jjMoveNfa_1(0, 0);
+   }
+}
+private final int jjMoveNfa_1(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 6;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0x9fffff7affffd9ffL & l) != 0L)
+                  {
+                     if (kind > 12)
+                        kind = 12;
+                     jjCheckNAdd(1);
+                  }
+                  else if ((0x100002600L & l) != 0L)
+                  {
+                     if (kind > 18)
+                        kind = 18;
+                     jjCheckNAdd(5);
+                  }
+                  else if (curChar == 61)
+                     jjstateSet[jjnewStateCnt++] = 3;
+                  else if (curChar == 62)
+                  {
+                     if (kind > 14)
+                        kind = 14;
+                  }
+                  break;
+               case 1:
+                  if ((0x9ffffffeffffd9ffL & l) == 0L)
+                     break;
+                  if (kind > 12)
+                     kind = 12;
+                  jjCheckNAdd(1);
+                  break;
+               case 2:
+               case 3:
+                  if (curChar == 62 && kind > 14)
+                     kind = 14;
+                  break;
+               case 4:
+                  if (curChar == 61)
+                     jjstateSet[jjnewStateCnt++] = 3;
+                  break;
+               case 5:
+                  if ((0x100002600L & l) == 0L)
+                     break;
+                  kind = 18;
+                  jjCheckNAdd(5);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+               case 1:
+                  if (kind > 12)
+                     kind = 12;
+                  jjCheckNAdd(1);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+               case 1:
+                  if ((jjbitVec0[i2] & l2) == 0L)
+                     break;
+                  if (kind > 12)
+                     kind = 12;
+                  jjCheckNAdd(1);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 6 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+static final int[] jjnextStates = {
+   17, 18, 21, 12, 14, 5, 8, 0, 4, 6, 0, 4, 6, 5, 0, 4, 
+   6, 12, 13, 
+};
+public static final String[] jjstrLiteralImages = {
+"", null, null, "\74\41\55\55", "\74\41", null, null, null, null, null, null, 
+null, null, "\75", null, null, "\47", "\42", null, null, null, null, null, null, 
+"\55\55\76", null, "\76", };
+public static final String[] lexStateNames = {
+   "DEFAULT", 
+   "WithinTag", 
+   "AfterEquals", 
+   "WithinQuote1", 
+   "WithinQuote2", 
+   "WithinComment1", 
+   "WithinComment2", 
+};
+public static final int[] jjnewLexState = {
+   -1, 1, 1, 5, 6, -1, -1, -1, -1, -1, -1, -1, -1, 2, 0, 1, 3, 4, -1, -1, 1, -1, 1, -1, 0, 
+   -1, 0, 
+};
+static final long[] jjtoToken = {
+   0x7fbfb3fL, 
+};
+static final long[] jjtoSkip = {
+   0x40000L, 
+};
+protected SimpleCharStream input_stream;
+private final int[] jjrounds = new int[25];
+private final int[] jjstateSet = new int[50];
+protected char curChar;
+public HTMLParserTokenManager(SimpleCharStream stream)
+{
+   if (SimpleCharStream.staticFlag)
+      throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.");
+   input_stream = stream;
+}
+public HTMLParserTokenManager(SimpleCharStream stream, int lexState)
+{
+   this(stream);
+   SwitchTo(lexState);
+}
+public void ReInit(SimpleCharStream stream)
+{
+   jjmatchedPos = jjnewStateCnt = 0;
+   curLexState = defaultLexState;
+   input_stream = stream;
+   ReInitRounds();
+}
+private final void ReInitRounds()
+{
+   int i;
+   jjround = 0x80000001;
+   for (i = 25; i-- > 0;)
+      jjrounds[i] = 0x80000000;
+}
+public void ReInit(SimpleCharStream stream, int lexState)
+{
+   ReInit(stream);
+   SwitchTo(lexState);
+}
+public void SwitchTo(int lexState)
+{
+   if (lexState >= 7 || lexState < 0)
+      throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE);
+   else
+      curLexState = lexState;
+}
+
+protected Token jjFillToken()
+{
+   Token t = Token.newToken(jjmatchedKind);
+   t.kind = jjmatchedKind;
+   String im = jjstrLiteralImages[jjmatchedKind];
+   t.image = (im == null) ? input_stream.GetImage() : im;
+   t.beginLine = input_stream.getBeginLine();
+   t.beginColumn = input_stream.getBeginColumn();
+   t.endLine = input_stream.getEndLine();
+   t.endColumn = input_stream.getEndColumn();
+   return t;
+}
+
+int curLexState = 0;
+int defaultLexState = 0;
+int jjnewStateCnt;
+int jjround;
+int jjmatchedPos;
+int jjmatchedKind;
+
+public Token getNextToken() 
+{
+  int kind;
+  Token specialToken = null;
+  Token matchedToken;
+  int curPos = 0;
+
+  EOFLoop :
+  for (;;)
+  {   
+   try   
+   {     
+      curChar = input_stream.BeginToken();
+   }     
+   catch(java.io.IOException e)
+   {        
+      jjmatchedKind = 0;
+      matchedToken = jjFillToken();
+      return matchedToken;
+   }
+
+   switch(curLexState)
+   {
+     case 0:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_0();
+       if (jjmatchedPos == 0 && jjmatchedKind > 11)
+       {
+          jjmatchedKind = 11;
+       }
+       break;
+     case 1:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_1();
+       break;
+     case 2:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_2();
+       break;
+     case 3:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_3();
+       break;
+     case 4:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_4();
+       break;
+     case 5:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_5();
+       break;
+     case 6:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_6();
+       break;
+   }
+     if (jjmatchedKind != 0x7fffffff)
+     {
+        if (jjmatchedPos + 1 < curPos)
+           input_stream.backup(curPos - jjmatchedPos - 1);
+        if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L)
+        {
+           matchedToken = jjFillToken();
+       if (jjnewLexState[jjmatchedKind] != -1)
+         curLexState = jjnewLexState[jjmatchedKind];
+           return matchedToken;
+        }
+        else
+        {
+         if (jjnewLexState[jjmatchedKind] != -1)
+           curLexState = jjnewLexState[jjmatchedKind];
+           continue EOFLoop;
+        }
+     }
+     int error_line = input_stream.getEndLine();
+     int error_column = input_stream.getEndColumn();
+     String error_after = null;
+     boolean EOFSeen = false;
+     try { input_stream.readChar(); input_stream.backup(1); }
+     catch (java.io.IOException e1) {
+        EOFSeen = true;
+        error_after = curPos <= 1 ? "" : input_stream.GetImage();
+        if (curChar == '\n' || curChar == '\r') {
+           error_line++;
+           error_column = 0;
+        }
+        else
+           error_column++;
+     }
+     if (!EOFSeen) {
+        input_stream.backup(1);
+        error_after = curPos <= 1 ? "" : input_stream.GetImage();
+     }
+     throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
+  }
+}
+
+}
Index: src/demo/org/apache/lucene/demo/html/ParseException.java
===================================================================
RCS file: src/demo/org/apache/lucene/demo/html/ParseException.java
diff -N src/demo/org/apache/lucene/demo/html/ParseException.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/demo/org/apache/lucene/demo/html/ParseException.java	14 Jun 2003 02:26:01 -0000
@@ -0,0 +1,192 @@
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 3.0 */
+package org.apache.lucene.demo.html;
+
+/**
+ * This exception is thrown when parse errors are encountered.
+ * You can explicitly create objects of this exception type by
+ * calling the method generateParseException in the generated
+ * parser.
+ *
+ * You can modify this class to customize your error reporting
+ * mechanisms so long as you retain the public fields.
+ */
+public class ParseException extends Exception {
+
+  /**
+   * This constructor is used by the method "generateParseException"
+   * in the generated parser.  Calling this constructor generates
+   * a new object of this type with the fields "currentToken",
+   * "expectedTokenSequences", and "tokenImage" set.  The boolean
+   * flag "specialConstructor" is also set to true to indicate that
+   * this constructor was used to create this object.
+   * This constructor calls its super class with the empty string
+   * to force the "toString" method of parent class "Throwable" to
+   * print the error message in the form:
+   *     ParseException: <result of getMessage>
+   */
+  public ParseException(Token currentTokenVal,
+                        int[][] expectedTokenSequencesVal,
+                        String[] tokenImageVal
+                       )
+  {
+    super("");
+    specialConstructor = true;
+    currentToken = currentTokenVal;
+    expectedTokenSequences = expectedTokenSequencesVal;
+    tokenImage = tokenImageVal;
+  }
+
+  /**
+   * The following constructors are for use by you for whatever
+   * purpose you can think of.  Constructing the exception in this
+   * manner makes the exception behave in the normal way - i.e., as
+   * documented in the class "Throwable".  The fields "errorToken",
+   * "expectedTokenSequences", and "tokenImage" do not contain
+   * relevant information.  The JavaCC generated code does not use
+   * these constructors.
+   */
+
+  public ParseException() {
+    super();
+    specialConstructor = false;
+  }
+
+  public ParseException(String message) {
+    super(message);
+    specialConstructor = false;
+  }
+
+  /**
+   * This variable determines which constructor was used to create
+   * this object and thereby affects the semantics of the
+   * "getMessage" method (see below).
+   */
+  protected boolean specialConstructor;
+
+  /**
+   * This is the last token that has been consumed successfully.  If
+   * this object has been created due to a parse error, the token
+   * followng this token will (therefore) be the first error token.
+   */
+  public Token currentToken;
+
+  /**
+   * Each entry in this array is an array of integers.  Each array
+   * of integers represents a sequence of tokens (by their ordinal
+   * values) that is expected at this point of the parse.
+   */
+  public int[][] expectedTokenSequences;
+
+  /**
+   * This is a reference to the "tokenImage" array of the generated
+   * parser within which the parse error occurred.  This array is
+   * defined in the generated ...Constants interface.
+   */
+  public String[] tokenImage;
+
+  /**
+   * This method has the standard behavior when this object has been
+   * created using the standard constructors.  Otherwise, it uses
+   * "currentToken" and "expectedTokenSequences" to generate a parse
+   * error message and returns it.  If this object has been created
+   * due to a parse error, and you do not catch it (it gets thrown
+   * from the parser), then this method is called during the printing
+   * of the final stack trace, and hence the correct error message
+   * gets displayed.
+   */
+  public String getMessage() {
+    if (!specialConstructor) {
+      return super.getMessage();
+    }
+    String expected = "";
+    int maxSize = 0;
+    for (int i = 0; i < expectedTokenSequences.length; i++) {
+      if (maxSize < expectedTokenSequences[i].length) {
+        maxSize = expectedTokenSequences[i].length;
+      }
+      for (int j = 0; j < expectedTokenSequences[i].length; j++) {
+        expected += tokenImage[expectedTokenSequences[i][j]] + " ";
+      }
+      if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
+        expected += "...";
+      }
+      expected += eol + "    ";
+    }
+    String retval = "Encountered \"";
+    Token tok = currentToken.next;
+    for (int i = 0; i < maxSize; i++) {
+      if (i != 0) retval += " ";
+      if (tok.kind == 0) {
+        retval += tokenImage[0];
+        break;
+      }
+      retval += add_escapes(tok.image);
+      tok = tok.next; 
+    }
+    retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
+    retval += "." + eol;
+    if (expectedTokenSequences.length == 1) {
+      retval += "Was expecting:" + eol + "    ";
+    } else {
+      retval += "Was expecting one of:" + eol + "    ";
+    }
+    retval += expected;
+    return retval;
+  }
+
+  /**
+   * The end of line string for this machine.
+   */
+  protected String eol = System.getProperty("line.separator", "\n");
+ 
+  /**
+   * Used to convert raw characters to their escaped version
+   * when these raw version cannot be used as part of an ASCII
+   * string literal.
+   */
+  protected String add_escapes(String str) {
+      StringBuffer retval = new StringBuffer();
+      char ch;
+      for (int i = 0; i < str.length(); i++) {
+        switch (str.charAt(i))
+        {
+           case 0 :
+              continue;
+           case '\b':
+              retval.append("\\b");
+              continue;
+           case '\t':
+              retval.append("\\t");
+              continue;
+           case '\n':
+              retval.append("\\n");
+              continue;
+           case '\f':
+              retval.append("\\f");
+              continue;
+           case '\r':
+              retval.append("\\r");
+              continue;
+           case '\"':
+              retval.append("\\\"");
+              continue;
+           case '\'':
+              retval.append("\\\'");
+              continue;
+           case '\\':
+              retval.append("\\\\");
+              continue;
+           default:
+              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+                 String s = "0000" + Integer.toString(ch, 16);
+                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+              } else {
+                 retval.append(ch);
+              }
+              continue;
+        }
+      }
+      return retval.toString();
+   }
+
+}
Index: src/demo/org/apache/lucene/demo/html/SimpleCharStream.java
===================================================================
RCS file: src/demo/org/apache/lucene/demo/html/SimpleCharStream.java
diff -N src/demo/org/apache/lucene/demo/html/SimpleCharStream.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/demo/org/apache/lucene/demo/html/SimpleCharStream.java	14 Jun 2003 02:26:02 -0000
@@ -0,0 +1,401 @@
+/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 3.0 */
+package org.apache.lucene.demo.html;
+
+/**
+ * An implementation of interface CharStream, where the stream is assumed to
+ * contain only ASCII characters (without unicode processing).
+ */
+
+public class SimpleCharStream
+{
+  public static final boolean staticFlag = false;
+  int bufsize;
+  int available;
+  int tokenBegin;
+  public int bufpos = -1;
+  protected int bufline[];
+  protected int bufcolumn[];
+
+  protected int column = 0;
+  protected int line = 1;
+
+  protected boolean prevCharIsCR = false;
+  protected boolean prevCharIsLF = false;
+
+  protected java.io.Reader inputStream;
+
+  protected char[] buffer;
+  protected int maxNextCharInd = 0;
+  protected int inBuf = 0;
+
+  protected void ExpandBuff(boolean wrapAround)
+  {
+     char[] newbuffer = new char[bufsize + 2048];
+     int newbufline[] = new int[bufsize + 2048];
+     int newbufcolumn[] = new int[bufsize + 2048];
+
+     try
+     {
+        if (wrapAround)
+        {
+           System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+           System.arraycopy(buffer, 0, newbuffer,
+                                             bufsize - tokenBegin, bufpos);
+           buffer = newbuffer;
+
+           System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+           System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
+           bufline = newbufline;
+
+           System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+           System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);
+           bufcolumn = newbufcolumn;
+
+           maxNextCharInd = (bufpos += (bufsize - tokenBegin));
+        }
+        else
+        {
+           System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+           buffer = newbuffer;
+
+           System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+           bufline = newbufline;
+
+           System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+           bufcolumn = newbufcolumn;
+
+           maxNextCharInd = (bufpos -= tokenBegin);
+        }
+     }
+     catch (Throwable t)
+     {
+        throw new Error(t.getMessage());
+     }
+
+
+     bufsize += 2048;
+     available = bufsize;
+     tokenBegin = 0;
+  }
+
+  protected void FillBuff() throws java.io.IOException
+  {
+     if (maxNextCharInd == available)
+     {
+        if (available == bufsize)
+        {
+           if (tokenBegin > 2048)
+           {
+              bufpos = maxNextCharInd = 0;
+              available = tokenBegin;
+           }
+           else if (tokenBegin < 0)
+              bufpos = maxNextCharInd = 0;
+           else
+              ExpandBuff(false);
+        }
+        else if (available > tokenBegin)
+           available = bufsize;
+        else if ((tokenBegin - available) < 2048)
+           ExpandBuff(true);
+        else
+           available = tokenBegin;
+     }
+
+     int i;
+     try {
+        if ((i = inputStream.read(buffer, maxNextCharInd,
+                                    available - maxNextCharInd)) == -1)
+        {
+           inputStream.close();
+           throw new java.io.IOException();
+        }
+        else
+           maxNextCharInd += i;
+        return;
+     }
+     catch(java.io.IOException e) {
+        --bufpos;
+        backup(0);
+        if (tokenBegin == -1)
+           tokenBegin = bufpos;
+        throw e;
+     }
+  }
+
+  public char BeginToken() throws java.io.IOException
+  {
+     tokenBegin = -1;
+     char c = readChar();
+     tokenBegin = bufpos;
+
+     return c;
+  }
+
+  protected void UpdateLineColumn(char c)
+  {
+     column++;
+
+     if (prevCharIsLF)
+     {
+        prevCharIsLF = false;
+        line += (column = 1);
+     }
+     else if (prevCharIsCR)
+     {
+        prevCharIsCR = false;
+        if (c == '\n')
+        {
+           prevCharIsLF = true;
+        }
+        else
+           line += (column = 1);
+     }
+
+     switch (c)
+     {
+        case '\r' :
+           prevCharIsCR = true;
+           break;
+        case '\n' :
+           prevCharIsLF = true;
+           break;
+        case '\t' :
+           column--;
+           column += (8 - (column & 07));
+           break;
+        default :
+           break;
+     }
+
+     bufline[bufpos] = line;
+     bufcolumn[bufpos] = column;
+  }
+
+  public char readChar() throws java.io.IOException
+  {
+     if (inBuf > 0)
+     {
+        --inBuf;
+
+        if (++bufpos == bufsize)
+           bufpos = 0;
+
+        return buffer[bufpos];
+     }
+
+     if (++bufpos >= maxNextCharInd)
+        FillBuff();
+
+     char c = buffer[bufpos];
+
+     UpdateLineColumn(c);
+     return (c);
+  }
+
+  /**
+   * @deprecated 
+   * @see #getEndColumn
+   */
+
+  public int getColumn() {
+     return bufcolumn[bufpos];
+  }
+
+  /**
+   * @deprecated 
+   * @see #getEndLine
+   */
+
+  public int getLine() {
+     return bufline[bufpos];
+  }
+
+  public int getEndColumn() {
+     return bufcolumn[bufpos];
+  }
+
+  public int getEndLine() {
+     return bufline[bufpos];
+  }
+
+  public int getBeginColumn() {
+     return bufcolumn[tokenBegin];
+  }
+
+  public int getBeginLine() {
+     return bufline[tokenBegin];
+  }
+
+  public void backup(int amount) {
+
+    inBuf += amount;
+    if ((bufpos -= amount) < 0)
+       bufpos += bufsize;
+  }
+
+  public SimpleCharStream(java.io.Reader dstream, int startline,
+  int startcolumn, int buffersize)
+  {
+    inputStream = dstream;
+    line = startline;
+    column = startcolumn - 1;
+
+    available = bufsize = buffersize;
+    buffer = new char[buffersize];
+    bufline = new int[buffersize];
+    bufcolumn = new int[buffersize];
+  }
+
+  public SimpleCharStream(java.io.Reader dstream, int startline,
+                                                           int startcolumn)
+  {
+     this(dstream, startline, startcolumn, 4096);
+  }
+
+  public SimpleCharStream(java.io.Reader dstream)
+  {
+     this(dstream, 1, 1, 4096);
+  }
+  public void ReInit(java.io.Reader dstream, int startline,
+  int startcolumn, int buffersize)
+  {
+    inputStream = dstream;
+    line = startline;
+    column = startcolumn - 1;
+
+    if (buffer == null || buffersize != buffer.length)
+    {
+      available = bufsize = buffersize;
+      buffer = new char[buffersize];
+      bufline = new int[buffersize];
+      bufcolumn = new int[buffersize];
+    }
+    prevCharIsLF = prevCharIsCR = false;
+    tokenBegin = inBuf = maxNextCharInd = 0;
+    bufpos = -1;
+  }
+
+  public void ReInit(java.io.Reader dstream, int startline,
+                                                           int startcolumn)
+  {
+     ReInit(dstream, startline, startcolumn, 4096);
+  }
+
+  public void ReInit(java.io.Reader dstream)
+  {
+     ReInit(dstream, 1, 1, 4096);
+  }
+  public SimpleCharStream(java.io.InputStream dstream, int startline,
+  int startcolumn, int buffersize)
+  {
+     this(new java.io.InputStreamReader(dstream), startline, startcolumn, 4096);
+  }
+
+  public SimpleCharStream(java.io.InputStream dstream, int startline,
+                                                           int startcolumn)
+  {
+     this(dstream, startline, startcolumn, 4096);
+  }
+
+  public SimpleCharStream(java.io.InputStream dstream)
+  {
+     this(dstream, 1, 1, 4096);
+  }
+
+  public void ReInit(java.io.InputStream dstream, int startline,
+                          int startcolumn, int buffersize)
+  {
+     ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, 4096);
+  }
+
+  public void ReInit(java.io.InputStream dstream)
+  {
+     ReInit(dstream, 1, 1, 4096);
+  }
+  public void ReInit(java.io.InputStream dstream, int startline,
+                                                           int startcolumn)
+  {
+     ReInit(dstream, startline, startcolumn, 4096);
+  }
+  public String GetImage()
+  {
+     if (bufpos >= tokenBegin)
+        return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
+     else
+        return new String(buffer, tokenBegin, bufsize - tokenBegin) +
+                              new String(buffer, 0, bufpos + 1);
+  }
+
+  public char[] GetSuffix(int len)
+  {
+     char[] ret = new char[len];
+
+     if ((bufpos + 1) >= len)
+        System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);
+     else
+     {
+        System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,
+                                                          len - bufpos - 1);
+        System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);
+     }
+
+     return ret;
+  }
+
+  public void Done()
+  {
+     buffer = null;
+     bufline = null;
+     bufcolumn = null;
+  }
+
+  /**
+   * Method to adjust line and column numbers for the start of a token.<BR>
+   */
+  public void adjustBeginLineColumn(int newLine, int newCol)
+  {
+     int start = tokenBegin;
+     int len;
+
+     if (bufpos >= tokenBegin)
+     {
+        len = bufpos - tokenBegin + inBuf + 1;
+     }
+     else
+     {
+        len = bufsize - tokenBegin + bufpos + 1 + inBuf;
+     }
+
+     int i = 0, j = 0, k = 0;
+     int nextColDiff = 0, columnDiff = 0;
+
+     while (i < len &&
+            bufline[j = start % bufsize] == bufline[k = ++start % bufsize])
+     {
+        bufline[j] = newLine;
+        nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];
+        bufcolumn[j] = newCol + columnDiff;
+        columnDiff = nextColDiff;
+        i++;
+     } 
+
+     if (i < len)
+     {
+        bufline[j] = newLine++;
+        bufcolumn[j] = newCol + columnDiff;
+
+        while (i++ < len)
+        {
+           if (bufline[j = start % bufsize] != bufline[++start % bufsize])
+              bufline[j] = newLine++;
+           else
+              bufline[j] = newLine;
+        }
+     }
+
+     line = bufline[j];
+     column = bufcolumn[j];
+  }
+
+}
Index: src/demo/org/apache/lucene/demo/html/Token.java
===================================================================
RCS file: src/demo/org/apache/lucene/demo/html/Token.java
diff -N src/demo/org/apache/lucene/demo/html/Token.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/demo/org/apache/lucene/demo/html/Token.java	14 Jun 2003 02:26:02 -0000
@@ -0,0 +1,81 @@
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 3.0 */
+package org.apache.lucene.demo.html;
+
+/**
+ * Describes the input token stream.
+ */
+
+public class Token {
+
+  /**
+   * An integer that describes the kind of this token.  This numbering
+   * system is determined by JavaCCParser, and a table of these numbers is
+   * stored in the file ...Constants.java.
+   */
+  public int kind;
+
+  /**
+   * beginLine and beginColumn describe the position of the first character
+   * of this token; endLine and endColumn describe the position of the
+   * last character of this token.
+   */
+  public int beginLine, beginColumn, endLine, endColumn;
+
+  /**
+   * The string image of the token.
+   */
+  public String image;
+
+  /**
+   * A reference to the next regular (non-special) token from the input
+   * stream.  If this is the last token from the input stream, or if the
+   * token manager has not read tokens beyond this one, this field is
+   * set to null.  This is true only if this token is also a regular
+   * token.  Otherwise, see below for a description of the contents of
+   * this field.
+   */
+  public Token next;
+
+  /**
+   * This field is used to access special tokens that occur prior to this
+   * token, but after the immediately preceding regular (non-special) token.
+   * If there are no such special tokens, this field is set to null.
+   * When there are more than one such special token, this field refers
+   * to the last of these special tokens, which in turn refers to the next
+   * previous special token through its specialToken field, and so on
+   * until the first special token (whose specialToken field is null).
+   * The next fields of special tokens refer to other special tokens that
+   * immediately follow it (without an intervening regular token).  If there
+   * is no such token, this field is null.
+   */
+  public Token specialToken;
+
+  /**
+   * Returns the image.
+   */
+  public String toString()
+  {
+     return image;
+  }
+
+  /**
+   * Returns a new Token object, by default. However, if you want, you
+   * can create and return subclass objects based on the value of ofKind.
+   * Simply add the cases to the switch for all those special cases.
+   * For example, if you have a subclass of Token called IDToken that
+   * you want to create if ofKind is ID, simlpy add something like :
+   *
+   *    case MyParserConstants.ID : return new IDToken();
+   *
+   * to the following switch statement. Then you can cast matchedToken
+   * variable to the appropriate type and use it in your lexical actions.
+   */
+  public static final Token newToken(int ofKind)
+  {
+     switch(ofKind)
+     {
+       default : return new Token();
+     }
+  }
+
+}
Index: src/demo/org/apache/lucene/demo/html/TokenMgrError.java
===================================================================
RCS file: src/demo/org/apache/lucene/demo/html/TokenMgrError.java
diff -N src/demo/org/apache/lucene/demo/html/TokenMgrError.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/demo/org/apache/lucene/demo/html/TokenMgrError.java	14 Jun 2003 02:26:02 -0000
@@ -0,0 +1,133 @@
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */
+package org.apache.lucene.demo.html;
+
+public class TokenMgrError extends Error
+{
+   /*
+    * Ordinals for various reasons why an Error of this type can be thrown.
+    */
+
+   /**
+    * Lexical error occured.
+    */
+   static final int LEXICAL_ERROR = 0;
+
+   /**
+    * An attempt wass made to create a second instance of a static token manager.
+    */
+   static final int STATIC_LEXER_ERROR = 1;
+
+   /**
+    * Tried to change to an invalid lexical state.
+    */
+   static final int INVALID_LEXICAL_STATE = 2;
+
+   /**
+    * Detected (and bailed out of) an infinite loop in the token manager.
+    */
+   static final int LOOP_DETECTED = 3;
+
+   /**
+    * Indicates the reason why the exception is thrown. It will have
+    * one of the above 4 values.
+    */
+   int errorCode;
+
+   /**
+    * Replaces unprintable characters by their espaced (or unicode escaped)
+    * equivalents in the given string
+    */
+   protected static final String addEscapes(String str) {
+      StringBuffer retval = new StringBuffer();
+      char ch;
+      for (int i = 0; i < str.length(); i++) {
+        switch (str.charAt(i))
+        {
+           case 0 :
+              continue;
+           case '\b':
+              retval.append("\\b");
+              continue;
+           case '\t':
+              retval.append("\\t");
+              continue;
+           case '\n':
+              retval.append("\\n");
+              continue;
+           case '\f':
+              retval.append("\\f");
+              continue;
+           case '\r':
+              retval.append("\\r");
+              continue;
+           case '\"':
+              retval.append("\\\"");
+              continue;
+           case '\'':
+              retval.append("\\\'");
+              continue;
+           case '\\':
+              retval.append("\\\\");
+              continue;
+           default:
+              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+                 String s = "0000" + Integer.toString(ch, 16);
+                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+              } else {
+                 retval.append(ch);
+              }
+              continue;
+        }
+      }
+      return retval.toString();
+   }
+
+   /**
+    * Returns a detailed message for the Error when it is thrown by the
+    * token manager to indicate a lexical error.
+    * Parameters : 
+    *    EOFSeen     : indicates if EOF caused the lexicl error
+    *    curLexState : lexical state in which this error occured
+    *    errorLine   : line number when the error occured
+    *    errorColumn : column number when the error occured
+    *    errorAfter  : prefix that was seen before this error occured
+    *    curchar     : the offending character
+    * Note: You can customize the lexical error message by modifying this method.
+    */
+   protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
+      return("Lexical error at line " +
+           errorLine + ", column " +
+           errorColumn + ".  Encountered: " +
+           (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") +
+           "after : \"" + addEscapes(errorAfter) + "\"");
+   }
+
+   /**
+    * You can also modify the body of this method to customize your error messages.
+    * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+    * of end-users concern, so you can return something like : 
+    *
+    *     "Internal Error : Please file a bug report .... "
+    *
+    * from this method for such cases in the release version of your parser.
+    */
+   public String getMessage() {
+      return super.getMessage();
+   }
+
+   /*
+    * Constructors of various flavors follow.
+    */
+
+   public TokenMgrError() {
+   }
+
+   public TokenMgrError(String message, int reason) {
+      super(message);
+      errorCode = reason;
+   }
+
+   public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
+      this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
+   }
+}
Index: src/java/org/apache/lucene/analysis/standard/.cvsignore
===================================================================
RCS file: src/java/org/apache/lucene/analysis/standard/.cvsignore
diff -N src/java/org/apache/lucene/analysis/standard/.cvsignore
--- src/java/org/apache/lucene/analysis/standard/.cvsignore	18 Sep 2001 16:29:51 -0000	1.1.1.1
+++ /dev/null	1 Jan 1970 00:00:00 -0000
@@ -1,6 +0,0 @@
-Token.java
-StandardTokenizer.java
-StandardTokenizerTokenManager.java
-TokenMgrError.java
-CharStream.java
-StandardTokenizerConstants.java
Index: src/java/org/apache/lucene/analysis/standard/CharStream.java
===================================================================
RCS file: src/java/org/apache/lucene/analysis/standard/CharStream.java
diff -N src/java/org/apache/lucene/analysis/standard/CharStream.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/analysis/standard/CharStream.java	14 Jun 2003 02:26:02 -0000
@@ -0,0 +1,110 @@
+/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 3.0 */
+package org.apache.lucene.analysis.standard;
+
+/**
+ * This interface describes a character stream that maintains line and
+ * column number positions of the characters.  It also has the capability
+ * to backup the stream to some extent.  An implementation of this
+ * interface is used in the TokenManager implementation generated by
+ * JavaCCParser.
+ *
+ * All the methods except backup can be implemented in any fashion. backup
+ * needs to be implemented correctly for the correct operation of the lexer.
+ * Rest of the methods are all used to get information like line number,
+ * column number and the String that constitutes a token and are not used
+ * by the lexer. Hence their implementation won't affect the generated lexer's
+ * operation.
+ */
+
+public interface CharStream {
+
+  /**
+   * Returns the next character from the selected input.  The method
+   * of selecting the input is the responsibility of the class
+   * implementing this interface.  Can throw any java.io.IOException.
+   */
+  char readChar() throws java.io.IOException;
+
+  /**
+   * Returns the column position of the character last read.
+   * @deprecated 
+   * @see #getEndColumn
+   */
+  int getColumn();
+
+  /**
+   * Returns the line number of the character last read.
+   * @deprecated 
+   * @see #getEndLine
+   */
+  int getLine();
+
+  /**
+   * Returns the column number of the last character for current token (being
+   * matched after the last call to BeginTOken).
+   */
+  int getEndColumn();
+
+  /**
+   * Returns the line number of the last character for current token (being
+   * matched after the last call to BeginTOken).
+   */
+  int getEndLine();
+
+  /**
+   * Returns the column number of the first character for current token (being
+   * matched after the last call to BeginTOken).
+   */
+  int getBeginColumn();
+
+  /**
+   * Returns the line number of the first character for current token (being
+   * matched after the last call to BeginTOken).
+   */
+  int getBeginLine();
+
+  /**
+   * Backs up the input stream by amount steps. Lexer calls this method if it
+   * had already read some characters, but could not use them to match a
+   * (longer) token. So, they will be used again as the prefix of the next
+   * token and it is the implemetation's responsibility to do this right.
+   */
+  void backup(int amount);
+
+  /**
+   * Returns the next character that marks the beginning of the next token.
+   * All characters must remain in the buffer between two successive calls
+   * to this method to implement backup correctly.
+   */
+  char BeginToken() throws java.io.IOException;
+
+  /**
+   * Returns a string made up of characters from the marked token beginning 
+   * to the current buffer position. Implementations have the choice of returning
+   * anything that they want to. For example, for efficiency, one might decide
+   * to just return null, which is a valid implementation.
+   */
+  String GetImage();
+
+  /**
+   * Returns an array of characters that make up the suffix of length 'len' for
+   * the currently matched token. This is used to build up the matched string
+   * for use in actions in the case of MORE. A simple and inefficient
+   * implementation of this is as follows :
+   *
+   *   {
+   *      String t = GetImage();
+   *      return t.substring(t.length() - len, t.length()).toCharArray();
+   *   }
+   */
+  char[] GetSuffix(int len);
+
+  /**
+   * The lexer calls this function to indicate that it is done with the stream
+   * and hence implementations can free any resources held by this class.
+   * Again, the body of this function can be just empty and it will not
+   * affect the lexer's operation.
+   */
+  void Done();
+
+}
Index: src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
===================================================================
RCS file: src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
diff -N src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java	14 Jun 2003 02:26:02 -0000
@@ -0,0 +1,195 @@
+/* Generated By:JavaCC: Do not edit this line. StandardTokenizer.java */
+package org.apache.lucene.analysis.standard;
+
+import java.io.*;
+
+/** A grammar-based tokenizer constructed with JavaCC.
+ *
+ * <p> This should be a good tokenizer for most European-language documents.
+ *
+ * <p>Many applications have specific tokenizer needs.  If this tokenizer does
+ * not suit your application, please consider copying this source code
+ * directory to your project and maintaining your own grammar-based tokenizer.
+ */
+public class StandardTokenizer extends org.apache.lucene.analysis.Tokenizer implements StandardTokenizerConstants {
+
+  /** Constructs a tokenizer for this Reader. */
+  public StandardTokenizer(Reader reader) {
+    this(new FastCharStream(reader));
+    this.input = reader;
+  }
+
+/** Returns the next token in the stream, or null at EOS.
+ * <p>The returned token's type is set to an element of {@link
+ * StandardTokenizerConstants#tokenImage}.
+ */
+  final public org.apache.lucene.analysis.Token next() throws ParseException, IOException {
+  Token token = null;
+    switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+    case ALPHANUM:
+      token = jj_consume_token(ALPHANUM);
+      break;
+    case APOSTROPHE:
+      token = jj_consume_token(APOSTROPHE);
+      break;
+    case ACRONYM:
+      token = jj_consume_token(ACRONYM);
+      break;
+    case COMPANY:
+      token = jj_consume_token(COMPANY);
+      break;
+    case EMAIL:
+      token = jj_consume_token(EMAIL);
+      break;
+    case HOST:
+      token = jj_consume_token(HOST);
+      break;
+    case NUM:
+      token = jj_consume_token(NUM);
+      break;
+    case 0:
+      token = jj_consume_token(0);
+      break;
+    default:
+      jj_la1[0] = jj_gen;
+      jj_consume_token(-1);
+      throw new ParseException();
+    }
+      if (token.kind == EOF) {
+        {if (true) return null;}
+      } else {
+        {if (true) return
+          new org.apache.lucene.analysis.Token(token.image,
+                                        token.beginColumn,token.endColumn,
+                                        tokenImage[token.kind]);}
+      }
+    throw new Error("Missing return statement in function");
+  }
+
+  public StandardTokenizerTokenManager token_source;
+  public Token token, jj_nt;
+  private int jj_ntk;
+  private int jj_gen;
+  final private int[] jj_la1 = new int[1];
+  static private int[] jj_la1_0;
+  static {
+      jj_la1_0();
+   }
+   private static void jj_la1_0() {
+      jj_la1_0 = new int[] {0xff,};
+   }
+
+  public StandardTokenizer(CharStream stream) {
+    token_source = new StandardTokenizerTokenManager(stream);
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 1; i++) jj_la1[i] = -1;
+  }
+
+  public void ReInit(CharStream stream) {
+    token_source.ReInit(stream);
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 1; i++) jj_la1[i] = -1;
+  }
+
+  public StandardTokenizer(StandardTokenizerTokenManager tm) {
+    token_source = tm;
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 1; i++) jj_la1[i] = -1;
+  }
+
+  public void ReInit(StandardTokenizerTokenManager tm) {
+    token_source = tm;
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 1; i++) jj_la1[i] = -1;
+  }
+
+  final private Token jj_consume_token(int kind) throws ParseException {
+    Token oldToken;
+    if ((oldToken = token).next != null) token = token.next;
+    else token = token.next = token_source.getNextToken();
+    jj_ntk = -1;
+    if (token.kind == kind) {
+      jj_gen++;
+      return token;
+    }
+    token = oldToken;
+    jj_kind = kind;
+    throw generateParseException();
+  }
+
+  final public Token getNextToken() {
+    if (token.next != null) token = token.next;
+    else token = token.next = token_source.getNextToken();
+    jj_ntk = -1;
+    jj_gen++;
+    return token;
+  }
+
+  final public Token getToken(int index) {
+    Token t = token;
+    for (int i = 0; i < index; i++) {
+      if (t.next != null) t = t.next;
+      else t = t.next = token_source.getNextToken();
+    }
+    return t;
+  }
+
+  final private int jj_ntk() {
+    if ((jj_nt=token.next) == null)
+      return (jj_ntk = (token.next=token_source.getNextToken()).kind);
+    else
+      return (jj_ntk = jj_nt.kind);
+  }
+
+  private java.util.Vector jj_expentries = new java.util.Vector();
+  private int[] jj_expentry;
+  private int jj_kind = -1;
+
+  public ParseException generateParseException() {
+    jj_expentries.removeAllElements();
+    boolean[] la1tokens = new boolean[14];
+    for (int i = 0; i < 14; i++) {
+      la1tokens[i] = false;
+    }
+    if (jj_kind >= 0) {
+      la1tokens[jj_kind] = true;
+      jj_kind = -1;
+    }
+    for (int i = 0; i < 1; i++) {
+      if (jj_la1[i] == jj_gen) {
+        for (int j = 0; j < 32; j++) {
+          if ((jj_la1_0[i] & (1<<j)) != 0) {
+            la1tokens[j] = true;
+          }
+        }
+      }
+    }
+    for (int i = 0; i < 14; i++) {
+      if (la1tokens[i]) {
+        jj_expentry = new int[1];
+        jj_expentry[0] = i;
+        jj_expentries.addElement(jj_expentry);
+      }
+    }
+    int[][] exptokseq = new int[jj_expentries.size()][];
+    for (int i = 0; i < jj_expentries.size(); i++) {
+      exptokseq[i] = (int[])jj_expentries.elementAt(i);
+    }
+    return new ParseException(token, exptokseq, tokenImage);
+  }
+
+  final public void enable_tracing() {
+  }
+
+  final public void disable_tracing() {
+  }
+
+}
Index: src/java/org/apache/lucene/analysis/standard/StandardTokenizerConstants.java
===================================================================
RCS file: src/java/org/apache/lucene/analysis/standard/StandardTokenizerConstants.java
diff -N src/java/org/apache/lucene/analysis/standard/StandardTokenizerConstants.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/analysis/standard/StandardTokenizerConstants.java	14 Jun 2003 02:26:02 -0000
@@ -0,0 +1,40 @@
+/* Generated By:JavaCC: Do not edit this line. StandardTokenizerConstants.java */
+package org.apache.lucene.analysis.standard;
+
+public interface StandardTokenizerConstants {
+
+  int EOF = 0;
+  int ALPHANUM = 1;
+  int APOSTROPHE = 2;
+  int ACRONYM = 3;
+  int COMPANY = 4;
+  int EMAIL = 5;
+  int HOST = 6;
+  int NUM = 7;
+  int P = 8;
+  int HAS_DIGIT = 9;
+  int ALPHA = 10;
+  int LETTER = 11;
+  int DIGIT = 12;
+  int NOISE = 13;
+
+  int DEFAULT = 0;
+
+  String[] tokenImage = {
+    "<EOF>",
+    "<ALPHANUM>",
+    "<APOSTROPHE>",
+    "<ACRONYM>",
+    "<COMPANY>",
+    "<EMAIL>",
+    "<HOST>",
+    "<NUM>",
+    "<P>",
+    "<HAS_DIGIT>",
+    "<ALPHA>",
+    "<LETTER>",
+    "<DIGIT>",
+    "<NOISE>",
+  };
+
+}
Index: src/java/org/apache/lucene/analysis/standard/StandardTokenizerTokenManager.java
===================================================================
RCS file: src/java/org/apache/lucene/analysis/standard/StandardTokenizerTokenManager.java
diff -N src/java/org/apache/lucene/analysis/standard/StandardTokenizerTokenManager.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/analysis/standard/StandardTokenizerTokenManager.java	14 Jun 2003 02:26:03 -0000
@@ -0,0 +1,1095 @@
+/* Generated By:JavaCC: Do not edit this line. StandardTokenizerTokenManager.java */
+package org.apache.lucene.analysis.standard;
+import java.io.*;
+
+public class StandardTokenizerTokenManager implements StandardTokenizerConstants
+{
+  public  java.io.PrintStream debugStream = System.out;
+  public  void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
+private final int jjMoveStringLiteralDfa0_0()
+{
+   return jjMoveNfa_0(0, 0);
+}
+private final void jjCheckNAdd(int state)
+{
+   if (jjrounds[state] != jjround)
+   {
+      jjstateSet[jjnewStateCnt++] = state;
+      jjrounds[state] = jjround;
+   }
+}
+private final void jjAddStates(int start, int end)
+{
+   do {
+      jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+   } while (start++ != end);
+}
+private final void jjCheckNAddTwoStates(int state1, int state2)
+{
+   jjCheckNAdd(state1);
+   jjCheckNAdd(state2);
+}
+private final void jjCheckNAddStates(int start, int end)
+{
+   do {
+      jjCheckNAdd(jjnextStates[start]);
+   } while (start++ != end);
+}
+private final void jjCheckNAddStates(int start)
+{
+   jjCheckNAdd(jjnextStates[start]);
+   jjCheckNAdd(jjnextStates[start + 1]);
+}
+static final long[] jjbitVec0 = {
+   0x1600L, 0x0L, 0x0L, 0x0L
+};
+static final long[] jjbitVec1 = {
+   0x0L, 0xffc000000000L, 0x0L, 0xffc000000000L
+};
+static final long[] jjbitVec2 = {
+   0x0L, 0x3ff00000000L, 0x0L, 0x3ff000000000000L
+};
+static final long[] jjbitVec3 = {
+   0x0L, 0xffc000000000L, 0x0L, 0xff8000000000L
+};
+static final long[] jjbitVec4 = {
+   0x0L, 0xffc000000000L, 0x0L, 0x0L
+};
+static final long[] jjbitVec5 = {
+   0x0L, 0x3ff0000L, 0x0L, 0x3ff0000L
+};
+static final long[] jjbitVec6 = {
+   0x0L, 0x3ffL, 0x0L, 0x0L
+};
+static final long[] jjbitVec7 = {
+   0x1ff00000fffffffeL, 0xffffffffffffc000L, 0xffffffffL, 0x600000000000000L
+};
+static final long[] jjbitVec9 = {
+   0x0L, 0x0L, 0x0L, 0xff7fffffff7fffffL
+};
+static final long[] jjbitVec10 = {
+   0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+static final long[] jjbitVec11 = {
+   0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffL, 0x0L
+};
+static final long[] jjbitVec12 = {
+   0xffffffffffffffffL, 0xffffffffffffffffL, 0x0L, 0x0L
+};
+static final long[] jjbitVec13 = {
+   0x3fffffffffffL, 0x0L, 0x0L, 0x0L
+};
+private final int jjMoveNfa_0(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 74;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0x3ff000000000000L & l) != 0L)
+                  {
+                     if (kind > 1)
+                        kind = 1;
+                     jjCheckNAddStates(0, 17);
+                  }
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddStates(18, 23);
+                  break;
+               case 1:
+               case 40:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(1, 2);
+                  break;
+               case 2:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAdd(3);
+                  break;
+               case 3:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAdd(3);
+                  break;
+               case 4:
+               case 49:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(4, 5);
+                  break;
+               case 5:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAdd(6);
+                  break;
+               case 6:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(6, 7);
+                  break;
+               case 7:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(8, 9);
+                  break;
+               case 8:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(8, 9);
+                  break;
+               case 9:
+               case 10:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(5, 10);
+                  break;
+               case 11:
+               case 62:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(11, 12);
+                  break;
+               case 12:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAdd(13);
+                  break;
+               case 13:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(13, 14);
+                  break;
+               case 14:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(15, 16);
+                  break;
+               case 15:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(15, 16);
+                  break;
+               case 16:
+               case 17:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(17, 18);
+                  break;
+               case 18:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAdd(19);
+                  break;
+               case 19:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(14, 19);
+                  break;
+               case 20:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAddStates(0, 17);
+                  break;
+               case 21:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAdd(21);
+                  break;
+               case 22:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddStates(24, 26);
+                  break;
+               case 23:
+                  if ((0x600000000000L & l) != 0L)
+                     jjCheckNAddStates(27, 29);
+                  break;
+               case 25:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddStates(30, 33);
+                  break;
+               case 27:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddStates(34, 36);
+                  break;
+               case 28:
+                  if (curChar != 46)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddTwoStates(28, 29);
+                  break;
+               case 29:
+                  if (curChar == 45)
+                     jjCheckNAdd(30);
+                  break;
+               case 30:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddStates(37, 39);
+                  break;
+               case 31:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(31, 32);
+                  break;
+               case 32:
+                  if (curChar == 46)
+                     jjCheckNAdd(33);
+                  break;
+               case 33:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 6)
+                     kind = 6;
+                  jjCheckNAddTwoStates(32, 33);
+                  break;
+               case 34:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(34, 35);
+                  break;
+               case 35:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(36, 37);
+                  break;
+               case 36:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(36, 37);
+                  break;
+               case 37:
+               case 38:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAdd(38);
+                  break;
+               case 39:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(39, 40);
+                  break;
+               case 41:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(41, 42);
+                  break;
+               case 42:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(43, 44);
+                  break;
+               case 43:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(43, 44);
+                  break;
+               case 44:
+               case 45:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(45, 46);
+                  break;
+               case 46:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAdd(47);
+                  break;
+               case 47:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(42, 47);
+                  break;
+               case 48:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(48, 49);
+                  break;
+               case 50:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(50, 51);
+                  break;
+               case 51:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(52, 53);
+                  break;
+               case 52:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(52, 53);
+                  break;
+               case 53:
+               case 54:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(54, 55);
+                  break;
+               case 55:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAdd(56);
+                  break;
+               case 56:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(56, 57);
+                  break;
+               case 57:
+                  if ((0xf00000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(58, 59);
+                  break;
+               case 58:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(58, 59);
+                  break;
+               case 59:
+               case 60:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(55, 60);
+                  break;
+               case 61:
+                  if ((0x3ff000000000000L & l) != 0L)
+                     jjCheckNAddTwoStates(61, 62);
+                  break;
+               case 65:
+                  if (curChar == 39)
+                     jjstateSet[jjnewStateCnt++] = 66;
+                  break;
+               case 68:
+                  if (curChar == 46)
+                     jjCheckNAdd(69);
+                  break;
+               case 70:
+                  if (curChar != 46)
+                     break;
+                  if (kind > 3)
+                     kind = 3;
+                  jjCheckNAdd(69);
+                  break;
+               case 72:
+                  if (curChar == 38)
+                     jjstateSet[jjnewStateCnt++] = 73;
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddStates(40, 45);
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                  {
+                     if (kind > 1)
+                        kind = 1;
+                     jjCheckNAddStates(0, 17);
+                  }
+                  break;
+               case 1:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjAddStates(46, 47);
+                  break;
+               case 2:
+                  if (curChar == 95)
+                     jjCheckNAdd(3);
+                  break;
+               case 3:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAdd(3);
+                  break;
+               case 4:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(4, 5);
+                  break;
+               case 5:
+                  if (curChar == 95)
+                     jjCheckNAdd(6);
+                  break;
+               case 6:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(6, 7);
+                  break;
+               case 7:
+                  if (curChar == 95)
+                     jjCheckNAddTwoStates(8, 9);
+                  break;
+               case 8:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(8, 9);
+                  break;
+               case 10:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(5, 10);
+                  break;
+               case 11:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjAddStates(48, 49);
+                  break;
+               case 12:
+                  if (curChar == 95)
+                     jjCheckNAdd(13);
+                  break;
+               case 13:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(13, 14);
+                  break;
+               case 14:
+                  if (curChar == 95)
+                     jjCheckNAddTwoStates(15, 16);
+                  break;
+               case 15:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(15, 16);
+                  break;
+               case 17:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjAddStates(50, 51);
+                  break;
+               case 18:
+                  if (curChar == 95)
+                     jjCheckNAdd(19);
+                  break;
+               case 19:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(14, 19);
+                  break;
+               case 20:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAddStates(0, 17);
+                  break;
+               case 21:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAdd(21);
+                  break;
+               case 22:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddStates(24, 26);
+                  break;
+               case 24:
+                  if (curChar == 95)
+                     jjCheckNAdd(25);
+                  break;
+               case 25:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddStates(30, 33);
+                  break;
+               case 26:
+                  if (curChar == 64)
+                     jjCheckNAdd(27);
+                  break;
+               case 27:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddStates(34, 36);
+                  break;
+               case 30:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddStates(37, 39);
+                  break;
+               case 31:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(31, 32);
+                  break;
+               case 33:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 6)
+                     kind = 6;
+                  jjCheckNAddTwoStates(32, 33);
+                  break;
+               case 34:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(34, 35);
+                  break;
+               case 35:
+                  if (curChar == 95)
+                     jjCheckNAddTwoStates(36, 37);
+                  break;
+               case 36:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(36, 37);
+                  break;
+               case 38:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjstateSet[jjnewStateCnt++] = 38;
+                  break;
+               case 39:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(39, 40);
+                  break;
+               case 41:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(41, 42);
+                  break;
+               case 42:
+                  if (curChar == 95)
+                     jjCheckNAddTwoStates(43, 44);
+                  break;
+               case 43:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(43, 44);
+                  break;
+               case 45:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjAddStates(52, 53);
+                  break;
+               case 46:
+                  if (curChar == 95)
+                     jjCheckNAdd(47);
+                  break;
+               case 47:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(42, 47);
+                  break;
+               case 48:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(48, 49);
+                  break;
+               case 50:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(50, 51);
+                  break;
+               case 51:
+                  if (curChar == 95)
+                     jjCheckNAddTwoStates(52, 53);
+                  break;
+               case 52:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(52, 53);
+                  break;
+               case 54:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(54, 55);
+                  break;
+               case 55:
+                  if (curChar == 95)
+                     jjCheckNAdd(56);
+                  break;
+               case 56:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(56, 57);
+                  break;
+               case 57:
+                  if (curChar == 95)
+                     jjCheckNAddTwoStates(58, 59);
+                  break;
+               case 58:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(58, 59);
+                  break;
+               case 60:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(55, 60);
+                  break;
+               case 61:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(61, 62);
+                  break;
+               case 63:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddStates(40, 45);
+                  break;
+               case 64:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(64, 65);
+                  break;
+               case 66:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 2)
+                     kind = 2;
+                  jjCheckNAddTwoStates(65, 66);
+                  break;
+               case 67:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(67, 68);
+                  break;
+               case 69:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjAddStates(54, 55);
+                  break;
+               case 71:
+                  if ((0x7fffffe07fffffeL & l) != 0L)
+                     jjCheckNAddTwoStates(71, 72);
+                  break;
+               case 72:
+                  if (curChar == 64)
+                     jjCheckNAdd(73);
+                  break;
+               case 73:
+                  if ((0x7fffffe07fffffeL & l) == 0L)
+                     break;
+                  if (kind > 4)
+                     kind = 4;
+                  jjCheckNAdd(73);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((jjbitVec2[i2] & l2) != 0L)
+                     jjCheckNAddStates(18, 23);
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                  {
+                     if (kind > 1)
+                        kind = 1;
+                     jjCheckNAddStates(0, 17);
+                  }
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddStates(40, 45);
+                  break;
+               case 1:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(1, 2);
+                  break;
+               case 3:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjstateSet[jjnewStateCnt++] = 3;
+                  break;
+               case 4:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(4, 5);
+                  break;
+               case 6:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjAddStates(56, 57);
+                  break;
+               case 8:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjAddStates(58, 59);
+                  break;
+               case 9:
+                  if ((jjbitVec2[i2] & l2) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(5, 10);
+                  break;
+               case 10:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(5, 10);
+                  break;
+               case 11:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(11, 12);
+                  break;
+               case 13:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(13, 14);
+                  break;
+               case 15:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjAddStates(60, 61);
+                  break;
+               case 16:
+                  if ((jjbitVec2[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(17, 18);
+                  break;
+               case 17:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(17, 18);
+                  break;
+               case 19:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(14, 19);
+                  break;
+               case 20:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAddStates(0, 17);
+                  break;
+               case 21:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 1)
+                     kind = 1;
+                  jjCheckNAdd(21);
+                  break;
+               case 22:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddStates(24, 26);
+                  break;
+               case 25:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddStates(30, 33);
+                  break;
+               case 27:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddStates(34, 36);
+                  break;
+               case 30:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 5)
+                     kind = 5;
+                  jjCheckNAddStates(37, 39);
+                  break;
+               case 31:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(31, 32);
+                  break;
+               case 33:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 6)
+                     kind = 6;
+                  jjCheckNAddTwoStates(32, 33);
+                  break;
+               case 34:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(34, 35);
+                  break;
+               case 36:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjAddStates(62, 63);
+                  break;
+               case 37:
+                  if ((jjbitVec2[i2] & l2) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAdd(38);
+                  break;
+               case 38:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAdd(38);
+                  break;
+               case 39:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(39, 40);
+                  break;
+               case 40:
+                  if ((jjbitVec2[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(1, 2);
+                  break;
+               case 41:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(41, 42);
+                  break;
+               case 43:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjAddStates(64, 65);
+                  break;
+               case 44:
+                  if ((jjbitVec2[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(45, 46);
+                  break;
+               case 45:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(45, 46);
+                  break;
+               case 47:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(42, 47);
+                  break;
+               case 48:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(48, 49);
+                  break;
+               case 49:
+                  if ((jjbitVec2[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(4, 5);
+                  break;
+               case 50:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(50, 51);
+                  break;
+               case 52:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjAddStates(66, 67);
+                  break;
+               case 53:
+                  if ((jjbitVec2[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(54, 55);
+                  break;
+               case 54:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(54, 55);
+                  break;
+               case 56:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjAddStates(68, 69);
+                  break;
+               case 58:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjAddStates(70, 71);
+                  break;
+               case 59:
+                  if ((jjbitVec2[i2] & l2) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(55, 60);
+                  break;
+               case 60:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 7)
+                     kind = 7;
+                  jjCheckNAddTwoStates(55, 60);
+                  break;
+               case 61:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(61, 62);
+                  break;
+               case 62:
+                  if ((jjbitVec2[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(11, 12);
+                  break;
+               case 63:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddStates(40, 45);
+                  break;
+               case 64:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(64, 65);
+                  break;
+               case 66:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 2)
+                     kind = 2;
+                  jjCheckNAddTwoStates(65, 66);
+                  break;
+               case 67:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(67, 68);
+                  break;
+               case 69:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjAddStates(54, 55);
+                  break;
+               case 71:
+                  if ((jjbitVec9[i2] & l2) != 0L)
+                     jjCheckNAddTwoStates(71, 72);
+                  break;
+               case 73:
+                  if ((jjbitVec9[i2] & l2) == 0L)
+                     break;
+                  if (kind > 4)
+                     kind = 4;
+                  jjstateSet[jjnewStateCnt++] = 73;
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 74 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+static final int[] jjnextStates = {
+   21, 22, 23, 24, 31, 32, 34, 35, 39, 40, 41, 42, 48, 49, 50, 51, 
+   61, 62, 1, 2, 4, 5, 11, 12, 22, 23, 24, 23, 24, 26, 23, 24, 
+   25, 26, 27, 28, 29, 28, 29, 30, 64, 65, 67, 68, 71, 72, 1, 2, 
+   11, 12, 17, 18, 45, 46, 69, 70, 6, 7, 8, 9, 15, 16, 36, 37, 
+   43, 44, 52, 53, 56, 57, 58, 59, 
+};
+public static final String[] jjstrLiteralImages = {
+"", null, null, null, null, null, null, null, null, null, null, null, null, 
+null, };
+public static final String[] lexStateNames = {
+   "DEFAULT", 
+};
+static final long[] jjtoToken = {
+   0xffL, 
+};
+static final long[] jjtoSkip = {
+   0x2000L, 
+};
+protected CharStream input_stream;
+private final int[] jjrounds = new int[74];
+private final int[] jjstateSet = new int[148];
+protected char curChar;
+public StandardTokenizerTokenManager(CharStream stream)
+{
+   input_stream = stream;
+}
+public StandardTokenizerTokenManager(CharStream stream, int lexState)
+{
+   this(stream);
+   SwitchTo(lexState);
+}
+public void ReInit(CharStream stream)
+{
+   jjmatchedPos = jjnewStateCnt = 0;
+   curLexState = defaultLexState;
+   input_stream = stream;
+   ReInitRounds();
+}
+private final void ReInitRounds()
+{
+   int i;
+   jjround = 0x80000001;
+   for (i = 74; i-- > 0;)
+      jjrounds[i] = 0x80000000;
+}
+public void ReInit(CharStream stream, int lexState)
+{
+   ReInit(stream);
+   SwitchTo(lexState);
+}
+public void SwitchTo(int lexState)
+{
+   if (lexState >= 1 || lexState < 0)
+      throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE);
+   else
+      curLexState = lexState;
+}
+
+protected Token jjFillToken()
+{
+   Token t = Token.newToken(jjmatchedKind);
+   t.kind = jjmatchedKind;
+   String im = jjstrLiteralImages[jjmatchedKind];
+   t.image = (im == null) ? input_stream.GetImage() : im;
+   t.beginLine = input_stream.getBeginLine();
+   t.beginColumn = input_stream.getBeginColumn();
+   t.endLine = input_stream.getEndLine();
+   t.endColumn = input_stream.getEndColumn();
+   return t;
+}
+
+int curLexState = 0;
+int defaultLexState = 0;
+int jjnewStateCnt;
+int jjround;
+int jjmatchedPos;
+int jjmatchedKind;
+
+public Token getNextToken() 
+{
+  int kind;
+  Token specialToken = null;
+  Token matchedToken;
+  int curPos = 0;
+
+  EOFLoop :
+  for (;;)
+  {   
+   try   
+   {     
+      curChar = input_stream.BeginToken();
+   }     
+   catch(java.io.IOException e)
+   {        
+      jjmatchedKind = 0;
+      matchedToken = jjFillToken();
+      return matchedToken;
+   }
+
+   jjmatchedKind = 0x7fffffff;
+   jjmatchedPos = 0;
+   curPos = jjMoveStringLiteralDfa0_0();
+   if (jjmatchedPos == 0 && jjmatchedKind > 13)
+   {
+      jjmatchedKind = 13;
+   }
+   if (jjmatchedKind != 0x7fffffff)
+   {
+      if (jjmatchedPos + 1 < curPos)
+         input_stream.backup(curPos - jjmatchedPos - 1);
+      if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L)
+      {
+         matchedToken = jjFillToken();
+         return matchedToken;
+      }
+      else
+      {
+         continue EOFLoop;
+      }
+   }
+   int error_line = input_stream.getEndLine();
+   int error_column = input_stream.getEndColumn();
+   String error_after = null;
+   boolean EOFSeen = false;
+   try { input_stream.readChar(); input_stream.backup(1); }
+   catch (java.io.IOException e1) {
+      EOFSeen = true;
+      error_after = curPos <= 1 ? "" : input_stream.GetImage();
+      if (curChar == '\n' || curChar == '\r') {
+         error_line++;
+         error_column = 0;
+      }
+      else
+         error_column++;
+   }
+   if (!EOFSeen) {
+      input_stream.backup(1);
+      error_after = curPos <= 1 ? "" : input_stream.GetImage();
+   }
+   throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
+  }
+}
+
+}
Index: src/java/org/apache/lucene/analysis/standard/Token.java
===================================================================
RCS file: src/java/org/apache/lucene/analysis/standard/Token.java
diff -N src/java/org/apache/lucene/analysis/standard/Token.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/analysis/standard/Token.java	14 Jun 2003 02:26:03 -0000
@@ -0,0 +1,81 @@
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 3.0 */
+package org.apache.lucene.analysis.standard;
+
+/**
+ * Describes the input token stream.
+ */
+
+public class Token {
+
+  /**
+   * An integer that describes the kind of this token.  This numbering
+   * system is determined by JavaCCParser, and a table of these numbers is
+   * stored in the file ...Constants.java.
+   */
+  public int kind;
+
+  /**
+   * beginLine and beginColumn describe the position of the first character
+   * of this token; endLine and endColumn describe the position of the
+   * last character of this token.
+   */
+  public int beginLine, beginColumn, endLine, endColumn;
+
+  /**
+   * The string image of the token.
+   */
+  public String image;
+
+  /**
+   * A reference to the next regular (non-special) token from the input
+   * stream.  If this is the last token from the input stream, or if the
+   * token manager has not read tokens beyond this one, this field is
+   * set to null.  This is true only if this token is also a regular
+   * token.  Otherwise, see below for a description of the contents of
+   * this field.
+   */
+  public Token next;
+
+  /**
+   * This field is used to access special tokens that occur prior to this
+   * token, but after the immediately preceding regular (non-special) token.
+   * If there are no such special tokens, this field is set to null.
+   * When there are more than one such special token, this field refers
+   * to the last of these special tokens, which in turn refers to the next
+   * previous special token through its specialToken field, and so on
+   * until the first special token (whose specialToken field is null).
+   * The next fields of special tokens refer to other special tokens that
+   * immediately follow it (without an intervening regular token).  If there
+   * is no such token, this field is null.
+   */
+  public Token specialToken;
+
+  /**
+   * Returns the image.
+   */
+  public String toString()
+  {
+     return image;
+  }
+
+  /**
+   * Returns a new Token object, by default. However, if you want, you
+   * can create and return subclass objects based on the value of ofKind.
+   * Simply add the cases to the switch for all those special cases.
+   * For example, if you have a subclass of Token called IDToken that
+   * you want to create if ofKind is ID, simlpy add something like :
+   *
+   *    case MyParserConstants.ID : return new IDToken();
+   *
+   * to the following switch statement. Then you can cast matchedToken
+   * variable to the appropriate type and use it in your lexical actions.
+   */
+  public static final Token newToken(int ofKind)
+  {
+     switch(ofKind)
+     {
+       default : return new Token();
+     }
+  }
+
+}
Index: src/java/org/apache/lucene/analysis/standard/TokenMgrError.java
===================================================================
RCS file: src/java/org/apache/lucene/analysis/standard/TokenMgrError.java
diff -N src/java/org/apache/lucene/analysis/standard/TokenMgrError.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/analysis/standard/TokenMgrError.java	14 Jun 2003 02:26:03 -0000
@@ -0,0 +1,133 @@
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */
+package org.apache.lucene.analysis.standard;
+
+public class TokenMgrError extends Error
+{
+   /*
+    * Ordinals for various reasons why an Error of this type can be thrown.
+    */
+
+   /**
+    * Lexical error occured.
+    */
+   static final int LEXICAL_ERROR = 0;
+
+   /**
+    * An attempt wass made to create a second instance of a static token manager.
+    */
+   static final int STATIC_LEXER_ERROR = 1;
+
+   /**
+    * Tried to change to an invalid lexical state.
+    */
+   static final int INVALID_LEXICAL_STATE = 2;
+
+   /**
+    * Detected (and bailed out of) an infinite loop in the token manager.
+    */
+   static final int LOOP_DETECTED = 3;
+
+   /**
+    * Indicates the reason why the exception is thrown. It will have
+    * one of the above 4 values.
+    */
+   int errorCode;
+
+   /**
+    * Replaces unprintable characters by their espaced (or unicode escaped)
+    * equivalents in the given string
+    */
+   protected static final String addEscapes(String str) {
+      StringBuffer retval = new StringBuffer();
+      char ch;
+      for (int i = 0; i < str.length(); i++) {
+        switch (str.charAt(i))
+        {
+           case 0 :
+              continue;
+           case '\b':
+              retval.append("\\b");
+              continue;
+           case '\t':
+              retval.append("\\t");
+              continue;
+           case '\n':
+              retval.append("\\n");
+              continue;
+           case '\f':
+              retval.append("\\f");
+              continue;
+           case '\r':
+              retval.append("\\r");
+              continue;
+           case '\"':
+              retval.append("\\\"");
+              continue;
+           case '\'':
+              retval.append("\\\'");
+              continue;
+           case '\\':
+              retval.append("\\\\");
+              continue;
+           default:
+              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+                 String s = "0000" + Integer.toString(ch, 16);
+                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+              } else {
+                 retval.append(ch);
+              }
+              continue;
+        }
+      }
+      return retval.toString();
+   }
+
+   /**
+    * Returns a detailed message for the Error when it is thrown by the
+    * token manager to indicate a lexical error.
+    * Parameters : 
+    *    EOFSeen     : indicates if EOF caused the lexicl error
+    *    curLexState : lexical state in which this error occured
+    *    errorLine   : line number when the error occured
+    *    errorColumn : column number when the error occured
+    *    errorAfter  : prefix that was seen before this error occured
+    *    curchar     : the offending character
+    * Note: You can customize the lexical error message by modifying this method.
+    */
+   protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
+      return("Lexical error at line " +
+           errorLine + ", column " +
+           errorColumn + ".  Encountered: " +
+           (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") +
+           "after : \"" + addEscapes(errorAfter) + "\"");
+   }
+
+   /**
+    * You can also modify the body of this method to customize your error messages.
+    * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+    * of end-users concern, so you can return something like : 
+    *
+    *     "Internal Error : Please file a bug report .... "
+    *
+    * from this method for such cases in the release version of your parser.
+    */
+   public String getMessage() {
+      return super.getMessage();
+   }
+
+   /*
+    * Constructors of various flavors follow.
+    */
+
+   public TokenMgrError() {
+   }
+
+   public TokenMgrError(String message, int reason) {
+      super(message);
+      errorCode = reason;
+   }
+
+   public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
+      this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
+   }
+}
Index: src/java/org/apache/lucene/queryParser/.cvsignore
===================================================================
RCS file: src/java/org/apache/lucene/queryParser/.cvsignore
diff -N src/java/org/apache/lucene/queryParser/.cvsignore
--- src/java/org/apache/lucene/queryParser/.cvsignore	18 Sep 2001 16:29:55 -0000	1.1.1.1
+++ /dev/null	1 Jan 1970 00:00:00 -0000
@@ -1,6 +0,0 @@
-QueryParser.java
-TokenMgrError.java
-ParseException.java
-Token.java
-TokenManager.java
-QueryParserConstants.java
Index: src/java/org/apache/lucene/queryParser/CharStream.java
===================================================================
RCS file: src/java/org/apache/lucene/queryParser/CharStream.java
diff -N src/java/org/apache/lucene/queryParser/CharStream.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/queryParser/CharStream.java	14 Jun 2003 02:26:03 -0000
@@ -0,0 +1,110 @@
+/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 3.0 */
+package org.apache.lucene.queryParser;
+
+/**
+ * This interface describes a character stream that maintains line and
+ * column number positions of the characters.  It also has the capability
+ * to backup the stream to some extent.  An implementation of this
+ * interface is used in the TokenManager implementation generated by
+ * JavaCCParser.
+ *
+ * All the methods except backup can be implemented in any fashion. backup
+ * needs to be implemented correctly for the correct operation of the lexer.
+ * Rest of the methods are all used to get information like line number,
+ * column number and the String that constitutes a token and are not used
+ * by the lexer. Hence their implementation won't affect the generated lexer's
+ * operation.
+ */
+
+public interface CharStream {
+
+  /**
+   * Returns the next character from the selected input.  The method
+   * of selecting the input is the responsibility of the class
+   * implementing this interface.  Can throw any java.io.IOException.
+   */
+  char readChar() throws java.io.IOException;
+
+  /**
+   * Returns the column position of the character last read.
+   * @deprecated 
+   * @see #getEndColumn
+   */
+  int getColumn();
+
+  /**
+   * Returns the line number of the character last read.
+   * @deprecated 
+   * @see #getEndLine
+   */
+  int getLine();
+
+  /**
+   * Returns the column number of the last character for current token (being
+   * matched after the last call to BeginTOken).
+   */
+  int getEndColumn();
+
+  /**
+   * Returns the line number of the last character for current token (being
+   * matched after the last call to BeginTOken).
+   */
+  int getEndLine();
+
+  /**
+   * Returns the column number of the first character for current token (being
+   * matched after the last call to BeginTOken).
+   */
+  int getBeginColumn();
+
+  /**
+   * Returns the line number of the first character for current token (being
+   * matched after the last call to BeginTOken).
+   */
+  int getBeginLine();
+
+  /**
+   * Backs up the input stream by amount steps. Lexer calls this method if it
+   * had already read some characters, but could not use them to match a
+   * (longer) token. So, they will be used again as the prefix of the next
+   * token and it is the implemetation's responsibility to do this right.
+   */
+  void backup(int amount);
+
+  /**
+   * Returns the next character that marks the beginning of the next token.
+   * All characters must remain in the buffer between two successive calls
+   * to this method to implement backup correctly.
+   */
+  char BeginToken() throws java.io.IOException;
+
+  /**
+   * Returns a string made up of characters from the marked token beginning 
+   * to the current buffer position. Implementations have the choice of returning
+   * anything that they want to. For example, for efficiency, one might decide
+   * to just return null, which is a valid implementation.
+   */
+  String GetImage();
+
+  /**
+   * Returns an array of characters that make up the suffix of length 'len' for
+   * the currently matched token. This is used to build up the matched string
+   * for use in actions in the case of MORE. A simple and inefficient
+   * implementation of this is as follows :
+   *
+   *   {
+   *      String t = GetImage();
+   *      return t.substring(t.length() - len, t.length()).toCharArray();
+   *   }
+   */
+  char[] GetSuffix(int len);
+
+  /**
+   * The lexer calls this function to indicate that it is done with the stream
+   * and hence implementations can free any resources held by this class.
+   * Again, the body of this function can be just empty and it will not
+   * affect the lexer's operation.
+   */
+  void Done();
+
+}
Index: src/java/org/apache/lucene/queryParser/ParseException.java
===================================================================
RCS file: src/java/org/apache/lucene/queryParser/ParseException.java
diff -N src/java/org/apache/lucene/queryParser/ParseException.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/queryParser/ParseException.java	14 Jun 2003 02:26:03 -0000
@@ -0,0 +1,192 @@
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 3.0 */
+package org.apache.lucene.queryParser;
+
+/**
+ * This exception is thrown when parse errors are encountered.
+ * You can explicitly create objects of this exception type by
+ * calling the method generateParseException in the generated
+ * parser.
+ *
+ * You can modify this class to customize your error reporting
+ * mechanisms so long as you retain the public fields.
+ */
+public class ParseException extends Exception {
+
+  /**
+   * This constructor is used by the method "generateParseException"
+   * in the generated parser.  Calling this constructor generates
+   * a new object of this type with the fields "currentToken",
+   * "expectedTokenSequences", and "tokenImage" set.  The boolean
+   * flag "specialConstructor" is also set to true to indicate that
+   * this constructor was used to create this object.
+   * This constructor calls its super class with the empty string
+   * to force the "toString" method of parent class "Throwable" to
+   * print the error message in the form:
+   *     ParseException: <result of getMessage>
+   */
+  public ParseException(Token currentTokenVal,
+                        int[][] expectedTokenSequencesVal,
+                        String[] tokenImageVal
+                       )
+  {
+    super("");
+    specialConstructor = true;
+    currentToken = currentTokenVal;
+    expectedTokenSequences = expectedTokenSequencesVal;
+    tokenImage = tokenImageVal;
+  }
+
+  /**
+   * The following constructors are for use by you for whatever
+   * purpose you can think of.  Constructing the exception in this
+   * manner makes the exception behave in the normal way - i.e., as
+   * documented in the class "Throwable".  The fields "errorToken",
+   * "expectedTokenSequences", and "tokenImage" do not contain
+   * relevant information.  The JavaCC generated code does not use
+   * these constructors.
+   */
+
+  public ParseException() {
+    super();
+    specialConstructor = false;
+  }
+
+  public ParseException(String message) {
+    super(message);
+    specialConstructor = false;
+  }
+
+  /**
+   * This variable determines which constructor was used to create
+   * this object and thereby affects the semantics of the
+   * "getMessage" method (see below).
+   */
+  protected boolean specialConstructor;
+
+  /**
+   * This is the last token that has been consumed successfully.  If
+   * this object has been created due to a parse error, the token
+   * followng this token will (therefore) be the first error token.
+   */
+  public Token currentToken;
+
+  /**
+   * Each entry in this array is an array of integers.  Each array
+   * of integers represents a sequence of tokens (by their ordinal
+   * values) that is expected at this point of the parse.
+   */
+  public int[][] expectedTokenSequences;
+
+  /**
+   * This is a reference to the "tokenImage" array of the generated
+   * parser within which the parse error occurred.  This array is
+   * defined in the generated ...Constants interface.
+   */
+  public String[] tokenImage;
+
+  /**
+   * This method has the standard behavior when this object has been
+   * created using the standard constructors.  Otherwise, it uses
+   * "currentToken" and "expectedTokenSequences" to generate a parse
+   * error message and returns it.  If this object has been created
+   * due to a parse error, and you do not catch it (it gets thrown
+   * from the parser), then this method is called during the printing
+   * of the final stack trace, and hence the correct error message
+   * gets displayed.
+   */
+  public String getMessage() {
+    if (!specialConstructor) {
+      return super.getMessage();
+    }
+    String expected = "";
+    int maxSize = 0;
+    for (int i = 0; i < expectedTokenSequences.length; i++) {
+      if (maxSize < expectedTokenSequences[i].length) {
+        maxSize = expectedTokenSequences[i].length;
+      }
+      for (int j = 0; j < expectedTokenSequences[i].length; j++) {
+        expected += tokenImage[expectedTokenSequences[i][j]] + " ";
+      }
+      if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
+        expected += "...";
+      }
+      expected += eol + "    ";
+    }
+    String retval = "Encountered \"";
+    Token tok = currentToken.next;
+    for (int i = 0; i < maxSize; i++) {
+      if (i != 0) retval += " ";
+      if (tok.kind == 0) {
+        retval += tokenImage[0];
+        break;
+      }
+      retval += add_escapes(tok.image);
+      tok = tok.next; 
+    }
+    retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
+    retval += "." + eol;
+    if (expectedTokenSequences.length == 1) {
+      retval += "Was expecting:" + eol + "    ";
+    } else {
+      retval += "Was expecting one of:" + eol + "    ";
+    }
+    retval += expected;
+    return retval;
+  }
+
+  /**
+   * The end of line string for this machine.
+   */
+  protected String eol = System.getProperty("line.separator", "\n");
+ 
+  /**
+   * Used to convert raw characters to their escaped version
+   * when these raw version cannot be used as part of an ASCII
+   * string literal.
+   */
+  protected String add_escapes(String str) {
+      StringBuffer retval = new StringBuffer();
+      char ch;
+      for (int i = 0; i < str.length(); i++) {
+        switch (str.charAt(i))
+        {
+           case 0 :
+              continue;
+           case '\b':
+              retval.append("\\b");
+              continue;
+           case '\t':
+              retval.append("\\t");
+              continue;
+           case '\n':
+              retval.append("\\n");
+              continue;
+           case '\f':
+              retval.append("\\f");
+              continue;
+           case '\r':
+              retval.append("\\r");
+              continue;
+           case '\"':
+              retval.append("\\\"");
+              continue;
+           case '\'':
+              retval.append("\\\'");
+              continue;
+           case '\\':
+              retval.append("\\\\");
+              continue;
+           default:
+              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+                 String s = "0000" + Integer.toString(ch, 16);
+                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+              } else {
+                 retval.append(ch);
+              }
+              continue;
+        }
+      }
+      return retval.toString();
+   }
+
+}
Index: src/java/org/apache/lucene/queryParser/QueryParser.java
===================================================================
RCS file: src/java/org/apache/lucene/queryParser/QueryParser.java
diff -N src/java/org/apache/lucene/queryParser/QueryParser.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/queryParser/QueryParser.java	14 Jun 2003 02:26:04 -0000
@@ -0,0 +1,1004 @@
+/* Generated By:JavaCC: Do not edit this line. QueryParser.java */
+package org.apache.lucene.queryParser;
+
+import java.util.Vector;
+import java.io.*;
+import java.text.*;
+import java.util.*;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.analysis.*;
+import org.apache.lucene.document.*;
+import org.apache.lucene.search.*;
+
+/**
+ * This class is generated by JavaCC.  The only method that clients should need
+ * to call is <a href="#parse">parse()</a>.
+ *
+ * The syntax for query strings is as follows:
+ * A Query is a series of clauses.
+ * A clause may be prefixed by:
+ * <ul>
+ * <li> a plus (<code>+</code>) or a minus (<code>-</code>) sign, indicating
+ * that the clause is required or prohibited respectively; or
+ * <li> a term followed by a colon, indicating the field to be searched.
+ * This enables one to construct queries which search multiple fields.
+ * </ul>
+ *
+ * A clause may be either:
+ * <ul>
+ * <li> a term, indicating all the documents that contain this term; or
+ * <li> a nested query, enclosed in parentheses.  Note that this may be used
+ * with a <code>+</code>/<code>-</code> prefix to require any of a set of
+ * terms.
+ * </ul>
+ *
+ * Thus, in BNF, the query grammar is:
+ * <pre>
+ *   Query  ::= ( Clause )*
+ *   Clause ::= ["+", "-"] [&lt;TERM&gt; ":"] ( &lt;TERM&gt; | "(" Query ")" )
+ * </pre>
+ *
+ * <p>
+ * Examples of appropriately formatted queries can be found in the <a
+ * href="http://jakarta.apache.org/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java">test cases</a>.
+ * </p>
+ *
+ * @author Brian Goetz
+ * @author Peter Halacsy
+ * @author Tatu Saloranta
+ */
+
+public class QueryParser implements QueryParserConstants {
+
+  private static final int CONJ_NONE   = 0;
+  private static final int CONJ_AND    = 1;
+  private static final int CONJ_OR     = 2;
+
+  private static final int MOD_NONE    = 0;
+  private static final int MOD_NOT     = 10;
+  private static final int MOD_REQ     = 11;
+
+  public static final int DEFAULT_OPERATOR_OR  = 0;
+  public static final int DEFAULT_OPERATOR_AND = 1;
+
+  /** The actual operator that parser uses to combine query terms */
+  private int operator = DEFAULT_OPERATOR_OR;
+
+  /**
+   * Whether terms of wildcard and prefix queries are to be automatically
+   * lower-cased or not.  Default is <code>true</code>.
+   */
+  boolean lowercaseWildcardTerms = true;
+
+  Analyzer analyzer;
+  String field;
+  int phraseSlop = 0;
+
+
+  /** Parses a query string, returning a {@link org.apache.lucene.search.Query}.
+   *  @param query	the query string to be parsed.
+   *  @param field	the default field for query terms.
+   *  @param analyzer   used to find terms in the query text.
+   *  @throws ParseException if the parsing fails
+   */
+  static public Query parse(String query, String field, Analyzer analyzer)
+       throws ParseException {
+    try {
+      QueryParser parser = new QueryParser(field, analyzer);
+      return parser.parse(query);
+    }
+    catch (TokenMgrError tme) {
+      throw new ParseException(tme.getMessage());
+    }
+  }
+
+  /** Constructs a query parser.
+   *  @param field	the default field for query terms.
+   *  @param analyzer   used to find terms in the query text.
+   */
+  public QueryParser(String f, Analyzer a) {
+    this(new FastCharStream(new StringReader("")));
+    analyzer = a;
+    field = f;
+  }
+
+  /** Parses a query string, returning a
+   * <a href="lucene.search.Query.html">Query</a>.
+   *  @param query	the query string to be parsed.
+   *  @throws ParseException if the parsing fails
+   *  @throws TokenMgrError if ther parsing fails
+   */
+  public Query parse(String query) throws ParseException, TokenMgrError {
+    ReInit(new FastCharStream(new StringReader(query)));
+    return Query(field);
+  }
+
+  /**
+   * Sets the default slop for phrases.  If zero, then exact phrase matches
+   * are required.  Default value is zero.
+   */
+  public void setPhraseSlop(int phraseSlop) {
+    this.phraseSlop = phraseSlop;
+  }
+
+  /**
+   * Gets the default slop for phrases.
+   */
+  public int getPhraseSlop() {
+    return phraseSlop;
+  }
+
+  /**
+   * Sets the boolean operator of the QueryParser.
+   * In classic mode (<code>DEFAULT_OPERATOR_OR</mode>) terms without any modifiers
+   * are considered optional: for example <code>capital of Hungary</code> is equal to
+   * <code>capital OR of OR Hungary</code>.<br/>
+   * In <code>DEFAULT_OPERATOR_AND</code> terms are considered to be in conjuction: the
+   * above mentioned query is parsed as <code>capital AND of AND Hungary</code>
+   */
+  public void setOperator(int operator) {
+    this.operator = operator;
+  }
+
+  public int getOperator() {
+    return operator;
+  }
+
+  public void setLowercaseWildcardTerms(boolean lowercaseWildcardTerms) {
+    this.lowercaseWildcardTerms = lowercaseWildcardTerms;
+  }
+
+  public boolean getLowercaseWildcardTerms() {
+    return lowercaseWildcardTerms;
+  }
+
+  protected void addClause(Vector clauses, int conj, int mods, Query q) {
+    boolean required, prohibited;
+
+    // If this term is introduced by AND, make the preceding term required,
+    // unless it's already prohibited
+    if (conj == CONJ_AND) {
+      BooleanClause c = (BooleanClause) clauses.elementAt(clauses.size()-1);
+      if (!c.prohibited)
+        c.required = true;
+    }
+
+    if (operator == DEFAULT_OPERATOR_AND && conj == CONJ_OR) {
+      // If this term is introduced by OR, make the preceding term optional,
+      // unless it's prohibited (that means we leave -a OR b but +a OR b-->a OR b)
+      // notice if the input is a OR b, first term is parsed as required; without
+      // this modification a OR b would parsed as +a OR b
+      BooleanClause c = (BooleanClause) clauses.elementAt(clauses.size()-1);
+      if (!c.prohibited)
+        c.required = false;
+    }
+
+    // We might have been passed a null query; the term might have been
+    // filtered away by the analyzer.
+    if (q == null)
+      return;
+
+    if (operator == DEFAULT_OPERATOR_OR) {
+      // We set REQUIRED if we're introduced by AND or +; PROHIBITED if
+      // introduced by NOT or -; make sure not to set both.
+      prohibited = (mods == MOD_NOT);
+      required = (mods == MOD_REQ);
+      if (conj == CONJ_AND && !prohibited) {
+        required = true;
+      }
+    } else {
+      // We set PROHIBITED if we're introduced by NOT or -; We set REQUIRED
+      // if not PROHIBITED and not introduced by OR
+      prohibited = (mods == MOD_NOT);
+      required   = (!prohibited && conj != CONJ_OR);
+    }
+    clauses.addElement(new BooleanClause(q, required, prohibited));
+  }
+
+  protected Query getFieldQuery(String field,
+                                Analyzer analyzer,
+                                String queryText) {
+    // Use the analyzer to get all the tokens, and then build a TermQuery,
+    // PhraseQuery, or nothing based on the term count
+
+    TokenStream source = analyzer.tokenStream(field,
+                                              new StringReader(queryText));
+    Vector v = new Vector();
+    org.apache.lucene.analysis.Token t;
+
+    while (true) {
+      try {
+        t = source.next();
+      }
+      catch (IOException e) {
+        t = null;
+      }
+      if (t == null)
+        break;
+      v.addElement(t.termText());
+    }
+    if (v.size() == 0)
+      return null;
+    else if (v.size() == 1)
+      return new TermQuery(new Term(field, (String) v.elementAt(0)));
+    else {
+      PhraseQuery q = new PhraseQuery();
+      q.setSlop(phraseSlop);
+      for (int i=0; i<v.size(); i++) {
+        q.add(new Term(field, (String) v.elementAt(i)));
+      }
+      return q;
+    }
+  }
+
+  protected Query getRangeQuery(String field,
+                                Analyzer analyzer,
+                                String part1,
+                                String part2,
+                                boolean inclusive)
+  {
+    boolean isDate = false, isNumber = false;
+
+    try {
+      DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT);
+      df.setLenient(true);
+      Date d1 = df.parse(part1);
+      Date d2 = df.parse(part2);
+      part1 = DateField.dateToString(d1);
+      part2 = DateField.dateToString(d2);
+      isDate = true;
+    }
+    catch (Exception e) { }
+
+    if (!isDate) {
+      // @@@ Add number support
+    }
+
+    return new RangeQuery(new Term(field, part1),
+                          new Term(field, part2),
+                          inclusive);
+  }
+
+  /**
+   * Factory method for generating query, given a set of clauses.
+   * By default creates a boolean query composed of clauses passed in.
+   *
+   * Can be overridden by extending classes, to modify query being
+   * returned.
+   *
+   * @param clauses Vector that contains {@link BooleanClause} instances
+   *    to join.
+   *
+   * @return Resulting {@link Query} object.
+   */
+  protected Query getBooleanQuery(Vector clauses)
+  {
+    BooleanQuery query = new BooleanQuery();
+    for (int i = 0; i < clauses.size(); i++) {
+        query.add((BooleanClause)clauses.elementAt(i));
+    }
+    return query;
+  }
+
+  /**
+   * Factory method for generating a query. Called when parser
+   * parses an input term token that contains one or more wildcard
+   * characters (? and *), but is not a prefix term token (one
+   * that has just a single * character at the end)
+   *<p>
+   * Depending on settings, prefix term may be lower-cased
+   * automatically. It will not go through the default Analyzer,
+   * however, since normal Analyzers are unlikely to work properly
+   * with wildcard templates.
+   *<p>
+   * Can be overridden by extending classes, to provide custom handling for
+   * wildcard queries, which may be necessary due to missing analyzer calls.
+   *
+   * @param field Name of the field query will use.
+   * @param termStr Term token that contains one or more wild card
+   *   characters (? or *), but is not simple prefix term
+   *
+   * @return Resulting {@link Query} built for the term
+   */
+  protected Query getWildcardQuery(String field, String termStr)
+  {
+    if (lowercaseWildcardTerms) {
+        termStr = termStr.toLowerCase();
+    }
+    Term t = new Term(field, termStr);
+    return new WildcardQuery(t);
+  }
+
+  /**
+   * Factory method for generating a query (similar to
+   * ({@link #getWildcardQuery}). Called when parser parses an input term
+   * token that uses prefix notation; that is, contains a single '*' wildcard
+   * character as its last character. Since this is a special case
+   * of generic wildcard term, and such a query can be optimized easily,
+   * this usually results in a different query object.
+   *<p>
+   * Depending on settings, a prefix term may be lower-cased
+   * automatically. It will not go through the default Analyzer,
+   * however, since normal Analyzers are unlikely to work properly
+   * with wildcard templates.
+   *<p>
+   * Can be overridden by extending classes, to provide custom handling for
+   * wild card queries, which may be necessary due to missing analyzer calls.
+   *
+   * @param field Name of the field query will use.
+   * @param termStr Term token to use for building term for the query
+   *    (<b>without</b> trailing '*' character!)
+   *
+   * @return Resulting {@link Query} built for the term
+   */
+  protected Query getPrefixQuery(String field, String termStr)
+  {
+    if (lowercaseWildcardTerms) {
+        termStr = termStr.toLowerCase();
+    }
+    Term t = new Term(field, termStr);
+    return new PrefixQuery(t);
+  }
+
+  /**
+   * Factory method for generating a query (similar to
+   * ({@link #getWildcardQuery}). Called when parser parses
+   * an input term token that has the fuzzy suffix (~) appended.
+   *
+   * @param field Name of the field query will use.
+   * @param termStr Term token to use for building term for the query
+   *
+   * @return Resulting {@link Query} built for the term
+   */
+  protected Query getFuzzyQuery(String field, String termStr)
+  {
+    Term t = new Term(field, termStr);
+    return new FuzzyQuery(t);
+  }
+
+  public static void main(String[] args) throws Exception {
+    QueryParser qp = new QueryParser("field",
+                           new org.apache.lucene.analysis.SimpleAnalyzer());
+    Query q = qp.parse(args[0]);
+    System.out.println(q.toString("field"));
+  }
+
+// *   Query  ::= ( Clause )*
+// *   Clause ::= ["+", "-"] [<TERM> ":"] ( <TERM> | "(" Query ")" )
+  final public int Conjunction() throws ParseException {
+  int ret = CONJ_NONE;
+    switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+    case AND:
+    case OR:
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case AND:
+        jj_consume_token(AND);
+            ret = CONJ_AND;
+        break;
+      case OR:
+        jj_consume_token(OR);
+              ret = CONJ_OR;
+        break;
+      default:
+        jj_la1[0] = jj_gen;
+        jj_consume_token(-1);
+        throw new ParseException();
+      }
+      break;
+    default:
+      jj_la1[1] = jj_gen;
+      ;
+    }
+    {if (true) return ret;}
+    throw new Error("Missing return statement in function");
+  }
+
+  final public int Modifiers() throws ParseException {
+  int ret = MOD_NONE;
+    switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+    case NOT:
+    case PLUS:
+    case MINUS:
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case PLUS:
+        jj_consume_token(PLUS);
+              ret = MOD_REQ;
+        break;
+      case MINUS:
+        jj_consume_token(MINUS);
+                 ret = MOD_NOT;
+        break;
+      case NOT:
+        jj_consume_token(NOT);
+               ret = MOD_NOT;
+        break;
+      default:
+        jj_la1[2] = jj_gen;
+        jj_consume_token(-1);
+        throw new ParseException();
+      }
+      break;
+    default:
+      jj_la1[3] = jj_gen;
+      ;
+    }
+    {if (true) return ret;}
+    throw new Error("Missing return statement in function");
+  }
+
+  final public Query Query(String field) throws ParseException {
+  Vector clauses = new Vector();
+  Query q, firstQuery=null;
+  int conj, mods;
+    mods = Modifiers();
+    q = Clause(field);
+    addClause(clauses, CONJ_NONE, mods, q);
+    if (mods == MOD_NONE)
+        firstQuery=q;
+    label_1:
+    while (true) {
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case AND:
+      case OR:
+      case NOT:
+      case PLUS:
+      case MINUS:
+      case LPAREN:
+      case QUOTED:
+      case TERM:
+      case PREFIXTERM:
+      case WILDTERM:
+      case RANGEIN_START:
+      case RANGEEX_START:
+      case NUMBER:
+        ;
+        break;
+      default:
+        jj_la1[4] = jj_gen;
+        break label_1;
+      }
+      conj = Conjunction();
+      mods = Modifiers();
+      q = Clause(field);
+      addClause(clauses, conj, mods, q);
+    }
+      if (clauses.size() == 1 && firstQuery != null)
+        {if (true) return firstQuery;}
+      else {
+        {if (true) return getBooleanQuery(clauses);}
+      }
+    throw new Error("Missing return statement in function");
+  }
+
+  final public Query Clause(String field) throws ParseException {
+  Query q;
+  Token fieldToken=null, boost=null;
+    if (jj_2_1(2)) {
+      fieldToken = jj_consume_token(TERM);
+      jj_consume_token(COLON);
+                                field = fieldToken.image;
+    } else {
+      ;
+    }
+    switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+    case QUOTED:
+    case TERM:
+    case PREFIXTERM:
+    case WILDTERM:
+    case RANGEIN_START:
+    case RANGEEX_START:
+    case NUMBER:
+      q = Term(field);
+      break;
+    case LPAREN:
+      jj_consume_token(LPAREN);
+      q = Query(field);
+      jj_consume_token(RPAREN);
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case CARAT:
+        jj_consume_token(CARAT);
+        boost = jj_consume_token(NUMBER);
+        break;
+      default:
+        jj_la1[5] = jj_gen;
+        ;
+      }
+      break;
+    default:
+      jj_la1[6] = jj_gen;
+      jj_consume_token(-1);
+      throw new ParseException();
+    }
+      if (boost != null) {
+        float f = (float)1.0;
+        try {
+          f = Float.valueOf(boost.image).floatValue();
+          q.setBoost(f);
+        } catch (Exception ignored) { }
+      }
+      {if (true) return q;}
+    throw new Error("Missing return statement in function");
+  }
+
+  final public Query Term(String field) throws ParseException {
+  Token term, boost=null, slop=null, goop1, goop2;
+  boolean prefix = false;
+  boolean wildcard = false;
+  boolean fuzzy = false;
+  boolean rangein = false;
+  Query q;
+    switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+    case TERM:
+    case PREFIXTERM:
+    case WILDTERM:
+    case NUMBER:
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case TERM:
+        term = jj_consume_token(TERM);
+        break;
+      case PREFIXTERM:
+        term = jj_consume_token(PREFIXTERM);
+                             prefix=true;
+        break;
+      case WILDTERM:
+        term = jj_consume_token(WILDTERM);
+                           wildcard=true;
+        break;
+      case NUMBER:
+        term = jj_consume_token(NUMBER);
+        break;
+      default:
+        jj_la1[7] = jj_gen;
+        jj_consume_token(-1);
+        throw new ParseException();
+      }
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case FUZZY:
+        jj_consume_token(FUZZY);
+                 fuzzy=true;
+        break;
+      default:
+        jj_la1[8] = jj_gen;
+        ;
+      }
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case CARAT:
+        jj_consume_token(CARAT);
+        boost = jj_consume_token(NUMBER);
+        switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+        case FUZZY:
+          jj_consume_token(FUZZY);
+                                          fuzzy=true;
+          break;
+        default:
+          jj_la1[9] = jj_gen;
+          ;
+        }
+        break;
+      default:
+        jj_la1[10] = jj_gen;
+        ;
+      }
+       if (wildcard) {
+         q = getWildcardQuery(field, term.image);
+       } else if (prefix) {
+         q = getPrefixQuery(field, term.image.substring
+                            (0, term.image.length()-1));
+       } else if (fuzzy) {
+         q = getFuzzyQuery(field, term.image);
+       } else {
+         q = getFieldQuery(field, analyzer, term.image);
+       }
+      break;
+    case RANGEIN_START:
+      jj_consume_token(RANGEIN_START);
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case RANGEIN_GOOP:
+        goop1 = jj_consume_token(RANGEIN_GOOP);
+        break;
+      case RANGEIN_QUOTED:
+        goop1 = jj_consume_token(RANGEIN_QUOTED);
+        break;
+      default:
+        jj_la1[11] = jj_gen;
+        jj_consume_token(-1);
+        throw new ParseException();
+      }
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case RANGEIN_TO:
+        jj_consume_token(RANGEIN_TO);
+        break;
+      default:
+        jj_la1[12] = jj_gen;
+        ;
+      }
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case RANGEIN_GOOP:
+        goop2 = jj_consume_token(RANGEIN_GOOP);
+        break;
+      case RANGEIN_QUOTED:
+        goop2 = jj_consume_token(RANGEIN_QUOTED);
+        break;
+      default:
+        jj_la1[13] = jj_gen;
+        jj_consume_token(-1);
+        throw new ParseException();
+      }
+      jj_consume_token(RANGEIN_END);
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case CARAT:
+        jj_consume_token(CARAT);
+        boost = jj_consume_token(NUMBER);
+        break;
+      default:
+        jj_la1[14] = jj_gen;
+        ;
+      }
+          if (goop1.kind == RANGEIN_QUOTED)
+            goop1.image = goop1.image.substring(1, goop1.image.length()-1);
+          if (goop2.kind == RANGEIN_QUOTED)
+            goop2.image = goop2.image.substring(1, goop2.image.length()-1);
+
+          q = getRangeQuery(field, analyzer, goop1.image, goop2.image, true);
+      break;
+    case RANGEEX_START:
+      jj_consume_token(RANGEEX_START);
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case RANGEEX_GOOP:
+        goop1 = jj_consume_token(RANGEEX_GOOP);
+        break;
+      case RANGEEX_QUOTED:
+        goop1 = jj_consume_token(RANGEEX_QUOTED);
+        break;
+      default:
+        jj_la1[15] = jj_gen;
+        jj_consume_token(-1);
+        throw new ParseException();
+      }
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case RANGEEX_TO:
+        jj_consume_token(RANGEEX_TO);
+        break;
+      default:
+        jj_la1[16] = jj_gen;
+        ;
+      }
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case RANGEEX_GOOP:
+        goop2 = jj_consume_token(RANGEEX_GOOP);
+        break;
+      case RANGEEX_QUOTED:
+        goop2 = jj_consume_token(RANGEEX_QUOTED);
+        break;
+      default:
+        jj_la1[17] = jj_gen;
+        jj_consume_token(-1);
+        throw new ParseException();
+      }
+      jj_consume_token(RANGEEX_END);
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case CARAT:
+        jj_consume_token(CARAT);
+        boost = jj_consume_token(NUMBER);
+        break;
+      default:
+        jj_la1[18] = jj_gen;
+        ;
+      }
+          if (goop1.kind == RANGEEX_QUOTED)
+            goop1.image = goop1.image.substring(1, goop1.image.length()-1);
+          if (goop2.kind == RANGEEX_QUOTED)
+            goop2.image = goop2.image.substring(1, goop2.image.length()-1);
+
+          q = getRangeQuery(field, analyzer, goop1.image, goop2.image, false);
+      break;
+    case QUOTED:
+      term = jj_consume_token(QUOTED);
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case SLOP:
+        slop = jj_consume_token(SLOP);
+        break;
+      default:
+        jj_la1[19] = jj_gen;
+        ;
+      }
+      switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
+      case CARAT:
+        jj_consume_token(CARAT);
+        boost = jj_consume_token(NUMBER);
+        break;
+      default:
+        jj_la1[20] = jj_gen;
+        ;
+      }
+         q = getFieldQuery(field, analyzer,
+                           term.image.substring(1, term.image.length()-1));
+         if (slop != null && q instanceof PhraseQuery) {
+           try {
+             int s = Float.valueOf(slop.image.substring(1)).intValue();
+             ((PhraseQuery) q).setSlop(s);
+           }
+           catch (Exception ignored) { }
+         }
+      break;
+    default:
+      jj_la1[21] = jj_gen;
+      jj_consume_token(-1);
+      throw new ParseException();
+    }
+    if (boost != null) {
+      float f = (float) 1.0;
+      try {
+        f = Float.valueOf(boost.image).floatValue();
+      }
+      catch (Exception ignored) {
+          /* Should this be handled somehow? (defaults to "no boost", if
+	   * boost number is invalid)
+	   */
+      }
+
+      // avoid boosting null queries, such as those caused by stop words
+      if (q != null) {
+        q.setBoost(f);
+      }
+    }
+    {if (true) return q;}
+    throw new Error("Missing return statement in function");
+  }
+
+  final private boolean jj_2_1(int xla) {
+    jj_la = xla; jj_lastpos = jj_scanpos = token;
+    boolean retval = !jj_3_1();
+    jj_save(0, xla);
+    return retval;
+  }
+
+  final private boolean jj_3_1() {
+    if (jj_scan_token(TERM)) return true;
+    if (jj_la == 0 && jj_scanpos == jj_lastpos) return false;
+    if (jj_scan_token(COLON)) return true;
+    if (jj_la == 0 && jj_scanpos == jj_lastpos) return false;
+    return false;
+  }
+
+  public QueryParserTokenManager token_source;
+  public Token token, jj_nt;
+  private int jj_ntk;
+  private Token jj_scanpos, jj_lastpos;
+  private int jj_la;
+  public boolean lookingAhead = false;
+  private boolean jj_semLA;
+  private int jj_gen;
+  final private int[] jj_la1 = new int[22];
+  static private int[] jj_la1_0;
+  static private int[] jj_la1_1;
+  static {
+      jj_la1_0();
+      jj_la1_1();
+   }
+   private static void jj_la1_0() {
+      jj_la1_0 = new int[] {0x180,0x180,0xe00,0xe00,0x1f31f80,0x8000,0x1f31000,0x1320000,0x40000,0x40000,0x8000,0x18000000,0x2000000,0x18000000,0x8000,0x80000000,0x20000000,0x80000000,0x8000,0x80000,0x8000,0x1f30000,};
+   }
+   private static void jj_la1_1() {
+      jj_la1_1 = new int[] {0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x1,0x0,0x1,0x0,0x0,0x0,0x0,};
+   }
+  final private JJCalls[] jj_2_rtns = new JJCalls[1];
+  private boolean jj_rescan = false;
+  private int jj_gc = 0;
+
+  public QueryParser(CharStream stream) {
+    token_source = new QueryParserTokenManager(stream);
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 22; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  public void ReInit(CharStream stream) {
+    token_source.ReInit(stream);
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 22; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  public QueryParser(QueryParserTokenManager tm) {
+    token_source = tm;
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 22; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  public void ReInit(QueryParserTokenManager tm) {
+    token_source = tm;
+    token = new Token();
+    jj_ntk = -1;
+    jj_gen = 0;
+    for (int i = 0; i < 22; i++) jj_la1[i] = -1;
+    for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
+  }
+
+  final private Token jj_consume_token(int kind) throws ParseException {
+    Token oldToken;
+    if ((oldToken = token).next != null) token = token.next;
+    else token = token.next = token_source.getNextToken();
+    jj_ntk = -1;
+    if (token.kind == kind) {
+      jj_gen++;
+      if (++jj_gc > 100) {
+        jj_gc = 0;
+        for (int i = 0; i < jj_2_rtns.length; i++) {
+          JJCalls c = jj_2_rtns[i];
+          while (c != null) {
+            if (c.gen < jj_gen) c.first = null;
+            c = c.next;
+          }
+        }
+      }
+      return token;
+    }
+    token = oldToken;
+    jj_kind = kind;
+    throw generateParseException();
+  }
+
+  final private boolean jj_scan_token(int kind) {
+    if (jj_scanpos == jj_lastpos) {
+      jj_la--;
+      if (jj_scanpos.next == null) {
+        jj_lastpos = jj_scanpos = jj_scanpos.next = token_source.getNextToken();
+      } else {
+        jj_lastpos = jj_scanpos = jj_scanpos.next;
+      }
+    } else {
+      jj_scanpos = jj_scanpos.next;
+    }
+    if (jj_rescan) {
+      int i = 0; Token tok = token;
+      while (tok != null && tok != jj_scanpos) { i++; tok = tok.next; }
+      if (tok != null) jj_add_error_token(kind, i);
+    }
+    return (jj_scanpos.kind != kind);
+  }
+
+  final public Token getNextToken() {
+    if (token.next != null) token = token.next;
+    else token = token.next = token_source.getNextToken();
+    jj_ntk = -1;
+    jj_gen++;
+    return token;
+  }
+
+  final public Token getToken(int index) {
+    Token t = lookingAhead ? jj_scanpos : token;
+    for (int i = 0; i < index; i++) {
+      if (t.next != null) t = t.next;
+      else t = t.next = token_source.getNextToken();
+    }
+    return t;
+  }
+
+  final private int jj_ntk() {
+    if ((jj_nt=token.next) == null)
+      return (jj_ntk = (token.next=token_source.getNextToken()).kind);
+    else
+      return (jj_ntk = jj_nt.kind);
+  }
+
+  private java.util.Vector jj_expentries = new java.util.Vector();
+  private int[] jj_expentry;
+  private int jj_kind = -1;
+  private int[] jj_lasttokens = new int[100];
+  private int jj_endpos;
+
+  private void jj_add_error_token(int kind, int pos) {
+    if (pos >= 100) return;
+    if (pos == jj_endpos + 1) {
+      jj_lasttokens[jj_endpos++] = kind;
+    } else if (jj_endpos != 0) {
+      jj_expentry = new int[jj_endpos];
+      for (int i = 0; i < jj_endpos; i++) {
+        jj_expentry[i] = jj_lasttokens[i];
+      }
+      boolean exists = false;
+      for (java.util.Enumeration enum = jj_expentries.elements(); enum.hasMoreElements();) {
+        int[] oldentry = (int[])(enum.nextElement());
+        if (oldentry.length == jj_expentry.length) {
+          exists = true;
+          for (int i = 0; i < jj_expentry.length; i++) {
+            if (oldentry[i] != jj_expentry[i]) {
+              exists = false;
+              break;
+            }
+          }
+          if (exists) break;
+        }
+      }
+      if (!exists) jj_expentries.addElement(jj_expentry);
+      if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind;
+    }
+  }
+
+  public ParseException generateParseException() {
+    jj_expentries.removeAllElements();
+    boolean[] la1tokens = new boolean[33];
+    for (int i = 0; i < 33; i++) {
+      la1tokens[i] = false;
+    }
+    if (jj_kind >= 0) {
+      la1tokens[jj_kind] = true;
+      jj_kind = -1;
+    }
+    for (int i = 0; i < 22; i++) {
+      if (jj_la1[i] == jj_gen) {
+        for (int j = 0; j < 32; j++) {
+          if ((jj_la1_0[i] & (1<<j)) != 0) {
+            la1tokens[j] = true;
+          }
+          if ((jj_la1_1[i] & (1<<j)) != 0) {
+            la1tokens[32+j] = true;
+          }
+        }
+      }
+    }
+    for (int i = 0; i < 33; i++) {
+      if (la1tokens[i]) {
+        jj_expentry = new int[1];
+        jj_expentry[0] = i;
+        jj_expentries.addElement(jj_expentry);
+      }
+    }
+    jj_endpos = 0;
+    jj_rescan_token();
+    jj_add_error_token(0, 0);
+    int[][] exptokseq = new int[jj_expentries.size()][];
+    for (int i = 0; i < jj_expentries.size(); i++) {
+      exptokseq[i] = (int[])jj_expentries.elementAt(i);
+    }
+    return new ParseException(token, exptokseq, tokenImage);
+  }
+
+  final public void enable_tracing() {
+  }
+
+  final public void disable_tracing() {
+  }
+
+  final private void jj_rescan_token() {
+    jj_rescan = true;
+    for (int i = 0; i < 1; i++) {
+      JJCalls p = jj_2_rtns[i];
+      do {
+        if (p.gen > jj_gen) {
+          jj_la = p.arg; jj_lastpos = jj_scanpos = p.first;
+          switch (i) {
+            case 0: jj_3_1(); break;
+          }
+        }
+        p = p.next;
+      } while (p != null);
+    }
+    jj_rescan = false;
+  }
+
+  final private void jj_save(int index, int xla) {
+    JJCalls p = jj_2_rtns[index];
+    while (p.gen > jj_gen) {
+      if (p.next == null) { p = p.next = new JJCalls(); break; }
+      p = p.next;
+    }
+    p.gen = jj_gen + xla - jj_la; p.first = token; p.arg = xla;
+  }
+
+  static final class JJCalls {
+    int gen;
+    Token first;
+    int arg;
+    JJCalls next;
+  }
+
+}
Index: src/java/org/apache/lucene/queryParser/QueryParserConstants.java
===================================================================
RCS file: src/java/org/apache/lucene/queryParser/QueryParserConstants.java
diff -N src/java/org/apache/lucene/queryParser/QueryParserConstants.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/queryParser/QueryParserConstants.java	14 Jun 2003 02:26:04 -0000
@@ -0,0 +1,80 @@
+/* Generated By:JavaCC: Do not edit this line. QueryParserConstants.java */
+package org.apache.lucene.queryParser;
+
+public interface QueryParserConstants {
+
+  int EOF = 0;
+  int _NUM_CHAR = 1;
+  int _ESCAPED_CHAR = 2;
+  int _TERM_START_CHAR = 3;
+  int _TERM_CHAR = 4;
+  int _WHITESPACE = 5;
+  int AND = 7;
+  int OR = 8;
+  int NOT = 9;
+  int PLUS = 10;
+  int MINUS = 11;
+  int LPAREN = 12;
+  int RPAREN = 13;
+  int COLON = 14;
+  int CARAT = 15;
+  int QUOTED = 16;
+  int TERM = 17;
+  int FUZZY = 18;
+  int SLOP = 19;
+  int PREFIXTERM = 20;
+  int WILDTERM = 21;
+  int RANGEIN_START = 22;
+  int RANGEEX_START = 23;
+  int NUMBER = 24;
+  int RANGEIN_TO = 25;
+  int RANGEIN_END = 26;
+  int RANGEIN_QUOTED = 27;
+  int RANGEIN_GOOP = 28;
+  int RANGEEX_TO = 29;
+  int RANGEEX_END = 30;
+  int RANGEEX_QUOTED = 31;
+  int RANGEEX_GOOP = 32;
+
+  int Boost = 0;
+  int RangeEx = 1;
+  int RangeIn = 2;
+  int DEFAULT = 3;
+
+  String[] tokenImage = {
+    "<EOF>",
+    "<_NUM_CHAR>",
+    "<_ESCAPED_CHAR>",
+    "<_TERM_START_CHAR>",
+    "<_TERM_CHAR>",
+    "<_WHITESPACE>",
+    "<token of kind 6>",
+    "<AND>",
+    "<OR>",
+    "<NOT>",
+    "\"+\"",
+    "\"-\"",
+    "\"(\"",
+    "\")\"",
+    "\":\"",
+    "\"^\"",
+    "<QUOTED>",
+    "<TERM>",
+    "\"~\"",
+    "<SLOP>",
+    "<PREFIXTERM>",
+    "<WILDTERM>",
+    "\"[\"",
+    "\"{\"",
+    "<NUMBER>",
+    "\"TO\"",
+    "\"]\"",
+    "<RANGEIN_QUOTED>",
+    "<RANGEIN_GOOP>",
+    "\"TO\"",
+    "\"}\"",
+    "<RANGEEX_QUOTED>",
+    "<RANGEEX_GOOP>",
+  };
+
+}
Index: src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java
===================================================================
RCS file: src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java
diff -N src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java	14 Jun 2003 02:26:05 -0000
@@ -0,0 +1,1038 @@
+/* Generated By:JavaCC: Do not edit this line. QueryParserTokenManager.java */
+package org.apache.lucene.queryParser;
+import java.util.Vector;
+import java.io.*;
+import java.text.*;
+import java.util.*;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.analysis.*;
+import org.apache.lucene.document.*;
+import org.apache.lucene.search.*;
+
+public class QueryParserTokenManager implements QueryParserConstants
+{
+  public  java.io.PrintStream debugStream = System.out;
+  public  void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
+private final int jjStopStringLiteralDfa_3(int pos, long active0)
+{
+   switch (pos)
+   {
+      default :
+         return -1;
+   }
+}
+private final int jjStartNfa_3(int pos, long active0)
+{
+   return jjMoveNfa_3(jjStopStringLiteralDfa_3(pos, active0), pos + 1);
+}
+private final int jjStopAtPos(int pos, int kind)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   return pos + 1;
+}
+private final int jjStartNfaWithStates_3(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_3(state, pos + 1);
+}
+private final int jjMoveStringLiteralDfa0_3()
+{
+   switch(curChar)
+   {
+      case 40:
+         return jjStopAtPos(0, 12);
+      case 41:
+         return jjStopAtPos(0, 13);
+      case 43:
+         return jjStopAtPos(0, 10);
+      case 45:
+         return jjStopAtPos(0, 11);
+      case 58:
+         return jjStopAtPos(0, 14);
+      case 91:
+         return jjStopAtPos(0, 22);
+      case 94:
+         return jjStopAtPos(0, 15);
+      case 123:
+         return jjStopAtPos(0, 23);
+      case 126:
+         return jjStartNfaWithStates_3(0, 18, 18);
+      default :
+         return jjMoveNfa_3(0, 0);
+   }
+}
+private final void jjCheckNAdd(int state)
+{
+   if (jjrounds[state] != jjround)
+   {
+      jjstateSet[jjnewStateCnt++] = state;
+      jjrounds[state] = jjround;
+   }
+}
+private final void jjAddStates(int start, int end)
+{
+   do {
+      jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+   } while (start++ != end);
+}
+private final void jjCheckNAddTwoStates(int state1, int state2)
+{
+   jjCheckNAdd(state1);
+   jjCheckNAdd(state2);
+}
+private final void jjCheckNAddStates(int start, int end)
+{
+   do {
+      jjCheckNAdd(jjnextStates[start]);
+   } while (start++ != end);
+}
+private final void jjCheckNAddStates(int start)
+{
+   jjCheckNAdd(jjnextStates[start]);
+   jjCheckNAdd(jjnextStates[start + 1]);
+}
+static final long[] jjbitVec0 = {
+   0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+static final long[] jjbitVec2 = {
+   0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
+};
+private final int jjMoveNfa_3(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 31;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0x7bffd0f8fffffdffL & l) != 0L)
+                  {
+                     if (kind > 17)
+                        kind = 17;
+                     jjCheckNAddStates(0, 6);
+                  }
+                  else if ((0x100000200L & l) != 0L)
+                  {
+                     if (kind > 6)
+                        kind = 6;
+                  }
+                  else if (curChar == 34)
+                     jjCheckNAdd(15);
+                  else if (curChar == 33)
+                  {
+                     if (kind > 9)
+                        kind = 9;
+                  }
+                  if (curChar == 38)
+                     jjstateSet[jjnewStateCnt++] = 4;
+                  break;
+               case 4:
+                  if (curChar == 38 && kind > 7)
+                     kind = 7;
+                  break;
+               case 5:
+                  if (curChar == 38)
+                     jjstateSet[jjnewStateCnt++] = 4;
+                  break;
+               case 13:
+                  if (curChar == 33 && kind > 9)
+                     kind = 9;
+                  break;
+               case 14:
+                  if (curChar == 34)
+                     jjCheckNAdd(15);
+                  break;
+               case 15:
+                  if ((0xfffffffbffffffffL & l) != 0L)
+                     jjCheckNAddTwoStates(15, 16);
+                  break;
+               case 16:
+                  if (curChar == 34 && kind > 16)
+                     kind = 16;
+                  break;
+               case 18:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 19)
+                     kind = 19;
+                  jjstateSet[jjnewStateCnt++] = 18;
+                  break;
+               case 19:
+                  if ((0x7bffd0f8fffffdffL & l) == 0L)
+                     break;
+                  if (kind > 17)
+                     kind = 17;
+                  jjCheckNAddStates(0, 6);
+                  break;
+               case 20:
+                  if ((0x7bffd0f8fffffdffL & l) == 0L)
+                     break;
+                  if (kind > 17)
+                     kind = 17;
+                  jjCheckNAddTwoStates(20, 21);
+                  break;
+               case 22:
+                  if ((0x84002f0600000000L & l) == 0L)
+                     break;
+                  if (kind > 17)
+                     kind = 17;
+                  jjCheckNAddTwoStates(20, 21);
+                  break;
+               case 23:
+                  if ((0x7bffd0f8fffffdffL & l) != 0L)
+                     jjCheckNAddStates(7, 9);
+                  break;
+               case 24:
+                  if (curChar == 42 && kind > 20)
+                     kind = 20;
+                  break;
+               case 26:
+                  if ((0x84002f0600000000L & l) != 0L)
+                     jjCheckNAddStates(7, 9);
+                  break;
+               case 27:
+                  if ((0xfbffd4f8fffffdffL & l) == 0L)
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(27, 28);
+                  break;
+               case 29:
+                  if ((0x84002f0600000000L & l) == 0L)
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(27, 28);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0x97ffffff97ffffffL & l) != 0L)
+                  {
+                     if (kind > 17)
+                        kind = 17;
+                     jjCheckNAddStates(0, 6);
+                  }
+                  else if (curChar == 126)
+                     jjstateSet[jjnewStateCnt++] = 18;
+                  if (curChar == 92)
+                     jjCheckNAddStates(10, 12);
+                  else if (curChar == 78)
+                     jjstateSet[jjnewStateCnt++] = 11;
+                  else if (curChar == 124)
+                     jjstateSet[jjnewStateCnt++] = 8;
+                  else if (curChar == 79)
+                     jjstateSet[jjnewStateCnt++] = 6;
+                  else if (curChar == 65)
+                     jjstateSet[jjnewStateCnt++] = 2;
+                  break;
+               case 1:
+                  if (curChar == 68 && kind > 7)
+                     kind = 7;
+                  break;
+               case 2:
+                  if (curChar == 78)
+                     jjstateSet[jjnewStateCnt++] = 1;
+                  break;
+               case 3:
+                  if (curChar == 65)
+                     jjstateSet[jjnewStateCnt++] = 2;
+                  break;
+               case 6:
+                  if (curChar == 82 && kind > 8)
+                     kind = 8;
+                  break;
+               case 7:
+                  if (curChar == 79)
+                     jjstateSet[jjnewStateCnt++] = 6;
+                  break;
+               case 8:
+                  if (curChar == 124 && kind > 8)
+                     kind = 8;
+                  break;
+               case 9:
+                  if (curChar == 124)
+                     jjstateSet[jjnewStateCnt++] = 8;
+                  break;
+               case 10:
+                  if (curChar == 84 && kind > 9)
+                     kind = 9;
+                  break;
+               case 11:
+                  if (curChar == 79)
+                     jjstateSet[jjnewStateCnt++] = 10;
+                  break;
+               case 12:
+                  if (curChar == 78)
+                     jjstateSet[jjnewStateCnt++] = 11;
+                  break;
+               case 15:
+                  jjAddStates(13, 14);
+                  break;
+               case 17:
+                  if (curChar == 126)
+                     jjstateSet[jjnewStateCnt++] = 18;
+                  break;
+               case 19:
+                  if ((0x97ffffff97ffffffL & l) == 0L)
+                     break;
+                  if (kind > 17)
+                     kind = 17;
+                  jjCheckNAddStates(0, 6);
+                  break;
+               case 20:
+                  if ((0x97ffffff97ffffffL & l) == 0L)
+                     break;
+                  if (kind > 17)
+                     kind = 17;
+                  jjCheckNAddTwoStates(20, 21);
+                  break;
+               case 21:
+                  if (curChar == 92)
+                     jjCheckNAddTwoStates(22, 22);
+                  break;
+               case 22:
+                  if ((0x6800000078000000L & l) == 0L)
+                     break;
+                  if (kind > 17)
+                     kind = 17;
+                  jjCheckNAddTwoStates(20, 21);
+                  break;
+               case 23:
+                  if ((0x97ffffff97ffffffL & l) != 0L)
+                     jjCheckNAddStates(7, 9);
+                  break;
+               case 25:
+                  if (curChar == 92)
+                     jjCheckNAddTwoStates(26, 26);
+                  break;
+               case 26:
+                  if ((0x6800000078000000L & l) != 0L)
+                     jjCheckNAddStates(7, 9);
+                  break;
+               case 27:
+                  if ((0x97ffffff97ffffffL & l) == 0L)
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(27, 28);
+                  break;
+               case 28:
+                  if (curChar == 92)
+                     jjCheckNAddTwoStates(29, 29);
+                  break;
+               case 29:
+                  if ((0x6800000078000000L & l) == 0L)
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(27, 28);
+                  break;
+               case 30:
+                  if (curChar == 92)
+                     jjCheckNAddStates(10, 12);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int hiByte = (int)(curChar >> 8);
+         int i1 = hiByte >> 6;
+         long l1 = 1L << (hiByte & 077);
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 17)
+                     kind = 17;
+                  jjCheckNAddStates(0, 6);
+                  break;
+               case 15:
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     jjAddStates(13, 14);
+                  break;
+               case 20:
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 17)
+                     kind = 17;
+                  jjCheckNAddTwoStates(20, 21);
+                  break;
+               case 23:
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     jjCheckNAddStates(7, 9);
+                  break;
+               case 27:
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(27, 28);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 31 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+private final int jjStopStringLiteralDfa_1(int pos, long active0)
+{
+   switch (pos)
+   {
+      case 0:
+         if ((active0 & 0x20000000L) != 0L)
+         {
+            jjmatchedKind = 32;
+            return 4;
+         }
+         return -1;
+      default :
+         return -1;
+   }
+}
+private final int jjStartNfa_1(int pos, long active0)
+{
+   return jjMoveNfa_1(jjStopStringLiteralDfa_1(pos, active0), pos + 1);
+}
+private final int jjStartNfaWithStates_1(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_1(state, pos + 1);
+}
+private final int jjMoveStringLiteralDfa0_1()
+{
+   switch(curChar)
+   {
+      case 84:
+         return jjMoveStringLiteralDfa1_1(0x20000000L);
+      case 125:
+         return jjStopAtPos(0, 30);
+      default :
+         return jjMoveNfa_1(0, 0);
+   }
+}
+private final int jjMoveStringLiteralDfa1_1(long active0)
+{
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) {
+      jjStopStringLiteralDfa_1(0, active0);
+      return 1;
+   }
+   switch(curChar)
+   {
+      case 79:
+         if ((active0 & 0x20000000L) != 0L)
+            return jjStartNfaWithStates_1(1, 29, 4);
+         break;
+      default :
+         break;
+   }
+   return jjStartNfa_1(0, active0);
+}
+private final int jjMoveNfa_1(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 5;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0xfffffffeffffffffL & l) != 0L)
+                  {
+                     if (kind > 32)
+                        kind = 32;
+                     jjCheckNAdd(4);
+                  }
+                  if ((0x100000200L & l) != 0L)
+                  {
+                     if (kind > 6)
+                        kind = 6;
+                  }
+                  else if (curChar == 34)
+                     jjCheckNAdd(2);
+                  break;
+               case 1:
+                  if (curChar == 34)
+                     jjCheckNAdd(2);
+                  break;
+               case 2:
+                  if ((0xfffffffbffffffffL & l) != 0L)
+                     jjCheckNAddTwoStates(2, 3);
+                  break;
+               case 3:
+                  if (curChar == 34 && kind > 31)
+                     kind = 31;
+                  break;
+               case 4:
+                  if ((0xfffffffeffffffffL & l) == 0L)
+                     break;
+                  if (kind > 32)
+                     kind = 32;
+                  jjCheckNAdd(4);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+               case 4:
+                  if ((0xdfffffffffffffffL & l) == 0L)
+                     break;
+                  if (kind > 32)
+                     kind = 32;
+                  jjCheckNAdd(4);
+                  break;
+               case 2:
+                  jjAddStates(15, 16);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int hiByte = (int)(curChar >> 8);
+         int i1 = hiByte >> 6;
+         long l1 = 1L << (hiByte & 077);
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+               case 4:
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 32)
+                     kind = 32;
+                  jjCheckNAdd(4);
+                  break;
+               case 2:
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     jjAddStates(15, 16);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 5 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+private final int jjMoveStringLiteralDfa0_0()
+{
+   return jjMoveNfa_0(0, 0);
+}
+private final int jjMoveNfa_0(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 3;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 24)
+                     kind = 24;
+                  jjAddStates(17, 18);
+                  break;
+               case 1:
+                  if (curChar == 46)
+                     jjCheckNAdd(2);
+                  break;
+               case 2:
+                  if ((0x3ff000000000000L & l) == 0L)
+                     break;
+                  if (kind > 24)
+                     kind = 24;
+                  jjCheckNAdd(2);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int hiByte = (int)(curChar >> 8);
+         int i1 = hiByte >> 6;
+         long l1 = 1L << (hiByte & 077);
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 3 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+private final int jjStopStringLiteralDfa_2(int pos, long active0)
+{
+   switch (pos)
+   {
+      case 0:
+         if ((active0 & 0x2000000L) != 0L)
+         {
+            jjmatchedKind = 28;
+            return 4;
+         }
+         return -1;
+      default :
+         return -1;
+   }
+}
+private final int jjStartNfa_2(int pos, long active0)
+{
+   return jjMoveNfa_2(jjStopStringLiteralDfa_2(pos, active0), pos + 1);
+}
+private final int jjStartNfaWithStates_2(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_2(state, pos + 1);
+}
+private final int jjMoveStringLiteralDfa0_2()
+{
+   switch(curChar)
+   {
+      case 84:
+         return jjMoveStringLiteralDfa1_2(0x2000000L);
+      case 93:
+         return jjStopAtPos(0, 26);
+      default :
+         return jjMoveNfa_2(0, 0);
+   }
+}
+private final int jjMoveStringLiteralDfa1_2(long active0)
+{
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) {
+      jjStopStringLiteralDfa_2(0, active0);
+      return 1;
+   }
+   switch(curChar)
+   {
+      case 79:
+         if ((active0 & 0x2000000L) != 0L)
+            return jjStartNfaWithStates_2(1, 25, 4);
+         break;
+      default :
+         break;
+   }
+   return jjStartNfa_2(0, active0);
+}
+private final int jjMoveNfa_2(int startState, int curPos)
+{
+   int[] nextStates;
+   int startsAt = 0;
+   jjnewStateCnt = 5;
+   int i = 1;
+   jjstateSet[0] = startState;
+   int j, kind = 0x7fffffff;
+   for (;;)
+   {
+      if (++jjround == 0x7fffffff)
+         ReInitRounds();
+      if (curChar < 64)
+      {
+         long l = 1L << curChar;
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+                  if ((0xfffffffeffffffffL & l) != 0L)
+                  {
+                     if (kind > 28)
+                        kind = 28;
+                     jjCheckNAdd(4);
+                  }
+                  if ((0x100000200L & l) != 0L)
+                  {
+                     if (kind > 6)
+                        kind = 6;
+                  }
+                  else if (curChar == 34)
+                     jjCheckNAdd(2);
+                  break;
+               case 1:
+                  if (curChar == 34)
+                     jjCheckNAdd(2);
+                  break;
+               case 2:
+                  if ((0xfffffffbffffffffL & l) != 0L)
+                     jjCheckNAddTwoStates(2, 3);
+                  break;
+               case 3:
+                  if (curChar == 34 && kind > 27)
+                     kind = 27;
+                  break;
+               case 4:
+                  if ((0xfffffffeffffffffL & l) == 0L)
+                     break;
+                  if (kind > 28)
+                     kind = 28;
+                  jjCheckNAdd(4);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else if (curChar < 128)
+      {
+         long l = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+               case 4:
+                  if ((0xffffffffdfffffffL & l) == 0L)
+                     break;
+                  if (kind > 28)
+                     kind = 28;
+                  jjCheckNAdd(4);
+                  break;
+               case 2:
+                  jjAddStates(15, 16);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      else
+      {
+         int hiByte = (int)(curChar >> 8);
+         int i1 = hiByte >> 6;
+         long l1 = 1L << (hiByte & 077);
+         int i2 = (curChar & 0xff) >> 6;
+         long l2 = 1L << (curChar & 077);
+         MatchLoop: do
+         {
+            switch(jjstateSet[--i])
+            {
+               case 0:
+               case 4:
+                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 28)
+                     kind = 28;
+                  jjCheckNAdd(4);
+                  break;
+               case 2:
+                  if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+                     jjAddStates(15, 16);
+                  break;
+               default : break;
+            }
+         } while(i != startsAt);
+      }
+      if (kind != 0x7fffffff)
+      {
+         jjmatchedKind = kind;
+         jjmatchedPos = curPos;
+         kind = 0x7fffffff;
+      }
+      ++curPos;
+      if ((i = jjnewStateCnt) == (startsAt = 5 - (jjnewStateCnt = startsAt)))
+         return curPos;
+      try { curChar = input_stream.readChar(); }
+      catch(java.io.IOException e) { return curPos; }
+   }
+}
+static final int[] jjnextStates = {
+   20, 23, 24, 27, 28, 25, 21, 23, 24, 25, 22, 26, 29, 15, 16, 2, 
+   3, 0, 1, 
+};
+private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
+{
+   switch(hiByte)
+   {
+      case 0:
+         return ((jjbitVec2[i2] & l2) != 0L);
+      default : 
+         if ((jjbitVec0[i1] & l1) != 0L)
+            return true;
+         return false;
+   }
+}
+public static final String[] jjstrLiteralImages = {
+"", null, null, null, null, null, null, null, null, null, "\53", "\55", "\50", 
+"\51", "\72", "\136", null, null, "\176", null, null, null, "\133", "\173", null, 
+"\124\117", "\135", null, null, "\124\117", "\175", null, null, };
+public static final String[] lexStateNames = {
+   "Boost", 
+   "RangeEx", 
+   "RangeIn", 
+   "DEFAULT", 
+};
+public static final int[] jjnewLexState = {
+   -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, 2, 1, 3, 
+   -1, 3, -1, -1, -1, 3, -1, -1, 
+};
+static final long[] jjtoToken = {
+   0x1ffffff81L, 
+};
+static final long[] jjtoSkip = {
+   0x40L, 
+};
+protected CharStream input_stream;
+private final int[] jjrounds = new int[31];
+private final int[] jjstateSet = new int[62];
+protected char curChar;
+public QueryParserTokenManager(CharStream stream)
+{
+   input_stream = stream;
+}
+public QueryParserTokenManager(CharStream stream, int lexState)
+{
+   this(stream);
+   SwitchTo(lexState);
+}
+public void ReInit(CharStream stream)
+{
+   jjmatchedPos = jjnewStateCnt = 0;
+   curLexState = defaultLexState;
+   input_stream = stream;
+   ReInitRounds();
+}
+private final void ReInitRounds()
+{
+   int i;
+   jjround = 0x80000001;
+   for (i = 31; i-- > 0;)
+      jjrounds[i] = 0x80000000;
+}
+public void ReInit(CharStream stream, int lexState)
+{
+   ReInit(stream);
+   SwitchTo(lexState);
+}
+public void SwitchTo(int lexState)
+{
+   if (lexState >= 4 || lexState < 0)
+      throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE);
+   else
+      curLexState = lexState;
+}
+
+protected Token jjFillToken()
+{
+   Token t = Token.newToken(jjmatchedKind);
+   t.kind = jjmatchedKind;
+   String im = jjstrLiteralImages[jjmatchedKind];
+   t.image = (im == null) ? input_stream.GetImage() : im;
+   t.beginLine = input_stream.getBeginLine();
+   t.beginColumn = input_stream.getBeginColumn();
+   t.endLine = input_stream.getEndLine();
+   t.endColumn = input_stream.getEndColumn();
+   return t;
+}
+
+int curLexState = 3;
+int defaultLexState = 3;
+int jjnewStateCnt;
+int jjround;
+int jjmatchedPos;
+int jjmatchedKind;
+
+public Token getNextToken() 
+{
+  int kind;
+  Token specialToken = null;
+  Token matchedToken;
+  int curPos = 0;
+
+  EOFLoop :
+  for (;;)
+  {   
+   try   
+   {     
+      curChar = input_stream.BeginToken();
+   }     
+   catch(java.io.IOException e)
+   {        
+      jjmatchedKind = 0;
+      matchedToken = jjFillToken();
+      return matchedToken;
+   }
+
+   switch(curLexState)
+   {
+     case 0:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_0();
+       break;
+     case 1:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_1();
+       break;
+     case 2:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_2();
+       break;
+     case 3:
+       jjmatchedKind = 0x7fffffff;
+       jjmatchedPos = 0;
+       curPos = jjMoveStringLiteralDfa0_3();
+       break;
+   }
+     if (jjmatchedKind != 0x7fffffff)
+     {
+        if (jjmatchedPos + 1 < curPos)
+           input_stream.backup(curPos - jjmatchedPos - 1);
+        if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L)
+        {
+           matchedToken = jjFillToken();
+       if (jjnewLexState[jjmatchedKind] != -1)
+         curLexState = jjnewLexState[jjmatchedKind];
+           return matchedToken;
+        }
+        else
+        {
+         if (jjnewLexState[jjmatchedKind] != -1)
+           curLexState = jjnewLexState[jjmatchedKind];
+           continue EOFLoop;
+        }
+     }
+     int error_line = input_stream.getEndLine();
+     int error_column = input_stream.getEndColumn();
+     String error_after = null;
+     boolean EOFSeen = false;
+     try { input_stream.readChar(); input_stream.backup(1); }
+     catch (java.io.IOException e1) {
+        EOFSeen = true;
+        error_after = curPos <= 1 ? "" : input_stream.GetImage();
+        if (curChar == '\n' || curChar == '\r') {
+           error_line++;
+           error_column = 0;
+        }
+        else
+           error_column++;
+     }
+     if (!EOFSeen) {
+        input_stream.backup(1);
+        error_after = curPos <= 1 ? "" : input_stream.GetImage();
+     }
+     throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
+  }
+}
+
+}
Index: src/java/org/apache/lucene/queryParser/Token.java
===================================================================
RCS file: src/java/org/apache/lucene/queryParser/Token.java
diff -N src/java/org/apache/lucene/queryParser/Token.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/queryParser/Token.java	14 Jun 2003 02:26:05 -0000
@@ -0,0 +1,81 @@
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 3.0 */
+package org.apache.lucene.queryParser;
+
+/**
+ * Describes the input token stream.
+ */
+
+public class Token {
+
+  /**
+   * An integer that describes the kind of this token.  This numbering
+   * system is determined by JavaCCParser, and a table of these numbers is
+   * stored in the file ...Constants.java.
+   */
+  public int kind;
+
+  /**
+   * beginLine and beginColumn describe the position of the first character
+   * of this token; endLine and endColumn describe the position of the
+   * last character of this token.
+   */
+  public int beginLine, beginColumn, endLine, endColumn;
+
+  /**
+   * The string image of the token.
+   */
+  public String image;
+
+  /**
+   * A reference to the next regular (non-special) token from the input
+   * stream.  If this is the last token from the input stream, or if the
+   * token manager has not read tokens beyond this one, this field is
+   * set to null.  This is true only if this token is also a regular
+   * token.  Otherwise, see below for a description of the contents of
+   * this field.
+   */
+  public Token next;
+
+  /**
+   * This field is used to access special tokens that occur prior to this
+   * token, but after the immediately preceding regular (non-special) token.
+   * If there are no such special tokens, this field is set to null.
+   * When there are more than one such special token, this field refers
+   * to the last of these special tokens, which in turn refers to the next
+   * previous special token through its specialToken field, and so on
+   * until the first special token (whose specialToken field is null).
+   * The next fields of special tokens refer to other special tokens that
+   * immediately follow it (without an intervening regular token).  If there
+   * is no such token, this field is null.
+   */
+  public Token specialToken;
+
+  /**
+   * Returns the image.
+   */
+  public String toString()
+  {
+     return image;
+  }
+
+  /**
+   * Returns a new Token object, by default. However, if you want, you
+   * can create and return subclass objects based on the value of ofKind.
+   * Simply add the cases to the switch for all those special cases.
+   * For example, if you have a subclass of Token called IDToken that
+   * you want to create if ofKind is ID, simlpy add something like :
+   *
+   *    case MyParserConstants.ID : return new IDToken();
+   *
+   * to the following switch statement. Then you can cast matchedToken
+   * variable to the appropriate type and use it in your lexical actions.
+   */
+  public static final Token newToken(int ofKind)
+  {
+     switch(ofKind)
+     {
+       default : return new Token();
+     }
+  }
+
+}
Index: src/java/org/apache/lucene/queryParser/TokenMgrError.java
===================================================================
RCS file: src/java/org/apache/lucene/queryParser/TokenMgrError.java
diff -N src/java/org/apache/lucene/queryParser/TokenMgrError.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/java/org/apache/lucene/queryParser/TokenMgrError.java	14 Jun 2003 02:26:05 -0000
@@ -0,0 +1,133 @@
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */
+package org.apache.lucene.queryParser;
+
+public class TokenMgrError extends Error
+{
+   /*
+    * Ordinals for various reasons why an Error of this type can be thrown.
+    */
+
+   /**
+    * Lexical error occured.
+    */
+   static final int LEXICAL_ERROR = 0;
+
+   /**
+    * An attempt wass made to create a second instance of a static token manager.
+    */
+   static final int STATIC_LEXER_ERROR = 1;
+
+   /**
+    * Tried to change to an invalid lexical state.
+    */
+   static final int INVALID_LEXICAL_STATE = 2;
+
+   /**
+    * Detected (and bailed out of) an infinite loop in the token manager.
+    */
+   static final int LOOP_DETECTED = 3;
+
+   /**
+    * Indicates the reason why the exception is thrown. It will have
+    * one of the above 4 values.
+    */
+   int errorCode;
+
+   /**
+    * Replaces unprintable characters by their espaced (or unicode escaped)
+    * equivalents in the given string
+    */
+   protected static final String addEscapes(String str) {
+      StringBuffer retval = new StringBuffer();
+      char ch;
+      for (int i = 0; i < str.length(); i++) {
+        switch (str.charAt(i))
+        {
+           case 0 :
+              continue;
+           case '\b':
+              retval.append("\\b");
+              continue;
+           case '\t':
+              retval.append("\\t");
+              continue;
+           case '\n':
+              retval.append("\\n");
+              continue;
+           case '\f':
+              retval.append("\\f");
+              continue;
+           case '\r':
+              retval.append("\\r");
+              continue;
+           case '\"':
+              retval.append("\\\"");
+              continue;
+           case '\'':
+              retval.append("\\\'");
+              continue;
+           case '\\':
+              retval.append("\\\\");
+              continue;
+           default:
+              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+                 String s = "0000" + Integer.toString(ch, 16);
+                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+              } else {
+                 retval.append(ch);
+              }
+              continue;
+        }
+      }
+      return retval.toString();
+   }
+
+   /**
+    * Returns a detailed message for the Error when it is thrown by the
+    * token manager to indicate a lexical error.
+    * Parameters : 
+    *    EOFSeen     : indicates if EOF caused the lexicl error
+    *    curLexState : lexical state in which this error occured
+    *    errorLine   : line number when the error occured
+    *    errorColumn : column number when the error occured
+    *    errorAfter  : prefix that was seen before this error occured
+    *    curchar     : the offending character
+    * Note: You can customize the lexical error message by modifying this method.
+    */
+   protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
+      return("Lexical error at line " +
+           errorLine + ", column " +
+           errorColumn + ".  Encountered: " +
+           (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") +
+           "after : \"" + addEscapes(errorAfter) + "\"");
+   }
+
+   /**
+    * You can also modify the body of this method to customize your error messages.
+    * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+    * of end-users concern, so you can return something like : 
+    *
+    *     "Internal Error : Please file a bug report .... "
+    *
+    * from this method for such cases in the release version of your parser.
+    */
+   public String getMessage() {
+      return super.getMessage();
+   }
+
+   /*
+    * Constructors of various flavors follow.
+    */
+
+   public TokenMgrError() {
+   }
+
+   public TokenMgrError(String message, int reason) {
+      super(message);
+      errorCode = reason;
+   }
+
+   public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
+      this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
+   }
+}