Index: E:/projects/lucene/trunk/common-build.xml
===================================================================
--- E:/projects/lucene/trunk/common-build.xml	(revision 559784)
+++ E:/projects/lucene/trunk/common-build.xml	(working copy)
@@ -7,16 +7,16 @@
     The ASF licenses this file to You under the Apache License, Version 2.0
     the "License"); you may not use this file except in compliance with
     the License.  You may obtain a copy of the License at
- 
+
         http://www.apache.org/licenses/LICENSE-2.0
- 
+
     Unless required by applicable law or agreed to in writing, software
     distributed under the License is distributed on an "AS IS" BASIS,
     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     See the License for the specific language governing permissions and
     limitations under the License.
  -->
-	
+
 <project name="common" xmlns:artifact="antlib:org.apache.maven.artifact.ant">
   <description>
     This file is designed for importing into a main build file, and not intended
@@ -58,6 +58,7 @@
   <property name="maven.dist.dir" location="dist/maven"/>
 
   <property name="javacc.home" location="${common.dir}"/>
+  <property name="jflex.home" location="${common.dir}"/>
 
   <property name="junit.output.dir" location="${build.dir}/test"/>
   <property name="junit.reports" location="${build.dir}/test/reports"/>
@@ -71,11 +72,11 @@
     <include name="FIND_NOTHING"/>
     <exclude name="**/*"/>
   </patternset>
-	
+
   <condition property="build-1-5-contrib">
      <equals arg1="1.5" arg2="${ant.java.version}" />
   </condition>
-  
+
   <property name="clover.db.dir" location="${build.dir}/test/clover/db"/>
   <property name="clover.report.dir" location="${build.dir}/test/clover/reports"/>
 
@@ -98,6 +99,12 @@
     />
 
   <available
+    property="jflex.present"
+    classname="JFlex.anttask.JFlexTask"
+    classpath="${jflex.home}/lib/JFlex.jar"
+    />
+
+  <available
     property="junit.present"
     classname="junit.framework.TestCase"
     />
@@ -106,7 +113,7 @@
     property="maven.ant.tasks.present"
     classname="org.apache.maven.artifact.ant.Pom"
   />
-	
+
   <target name="clean"
     description="Removes contents of build and dist directories">
     <delete dir="${build.dir}"/>
@@ -129,11 +136,25 @@
       One or more of the JavaCC .jj files is newer than its corresponding
       .java file.  Run the "javacc" target to regenerate the artifacts.
     </echo>
+  </target>
+
+  <target name="jflex-uptodate-check">
+    <uptodate property="jflex.files.uptodate">
+      <srcfiles dir="src" includes="**/*.jflex" />
+      <mapper type="glob" from="*.jflex" to="*.java"/>
+    </uptodate>
   </target>
 
-  <target name="init" depends="javacc-uptodate-check, javacc-notice">
+  <target name="jflex-notice" unless="jflex.files.uptodate">
+    <echo>
+      One or more of the JFlex .jflex files is newer than its corresponding
+      .java file.  Run the "jflex" target to regenerate the artifacts.
+    </echo>
   </target>
 
+  <target name="init" depends="javacc-uptodate-check, javacc-notice, jflex-uptodate-check, jflex-notice">
+  </target>
+
   <target name="javacc-check">
     <fail unless="javacc.present">
       ##################################################################
@@ -161,7 +182,33 @@
     </fail>
 
   </target>
-	
+
+  <target name="jflex-check">
+    <fail unless="jflex.present">
+      ##################################################################
+      JFlex not found.
+      JFlex Home: ${jflex.home}
+
+      Please download and install JFlex from:
+
+      &lt;http://jflex.de/download.html&gt;
+
+      Then, create a build.properties file either in your home
+      directory, or within the Lucene directory and set the jflex.home
+      property to the path where JFlex is installed. For example,
+      if you installed JFlex in /usr/local/java/jflex-1.4.1, then set the
+      jflex.home property to:
+
+      jflex.home=/usr/local/java/jflex-1.4.1
+
+      If you get an error like the one below, then you have not installed
+      things correctly. Please check all your paths and try again.
+
+      ##################################################################
+    </fail>
+
+  </target>
+
   <target name="compile-core" depends="init, clover"
           description="Compiles core classes">
     <compile
@@ -179,12 +226,12 @@
     description="Packages the JAR file">
     <jarify />
   </target>
-	
+
   <target name="maven.ant.tasks-check">
 	<fail unless="maven.ant.tasks.present">
 	  ##################################################################
       Maven ant tasks not found.
-      Please make sure the maven-ant-tasks jar is in ANT_HOME/lib, or made 
+      Please make sure the maven-ant-tasks jar is in ANT_HOME/lib, or made
       available to Ant using other mechanisms like -lib or CLASSPATH.
       ##################################################################
   	</fail>
@@ -196,7 +243,7 @@
     <attribute name="pom.xml" default="${pom.xml}"/>
     <sequential>
       <copy file="@{pom.xml}" tofile="${build.dir}/@{pom.xml}">
-        <filterset begintoken="@" endtoken="@"> 
+        <filterset begintoken="@" endtoken="@">
           <filter token="version" value="${version}"/>
         </filterset>
       </copy>
@@ -206,9 +253,9 @@
         <pom refid="maven.project"/>
       </artifact:install>
     </sequential>
-  </macrodef>  	
-	
-	
+  </macrodef>
+
+
   <macrodef name="jarify" description="Builds a JAR file">
     <attribute name="title" default="Lucene Search Engine: ${ant.project.name}" />
     <element name="manifest-attributes" optional="yes"/>
@@ -218,7 +265,7 @@
             outputproperty="svnversion" failifexecutionfails="false">
         <arg line="."/>
       </exec>
-     
+
       <jar
         destfile="${build.dir}/${final.name}.jar"
         basedir="${build.dir}/classes/java">
@@ -232,14 +279,14 @@
           -->
           <!-- Don't set 'Manifest-Version' it identifies the version of the
                manifest file format, and should allways be 1.0 (the default)
-  
-               Don't set 'Created-by' attribute, it's purpose is 
+
+               Don't set 'Created-by' attribute, it's purpose is
                to identify the version of java used to build the jar,
                which ant will do by default.
-  
+
                Ant will happily override these with bogus strings if you
                tell it to, so don't.
-  
+
                NOTE: we don't use section info because all of our manifest data
                applies to the entire jar/war ... no package specific info.
           -->
@@ -254,9 +301,9 @@
                      value="${version} ${svnversion} - ${user.name} - ${DSTAMP} ${TSTAMP}"/>
           <attribute name="Implementation-Vendor"
                      value="The Apache Software Foundation"/>
-          <attribute name="X-Compile-Source-JDK" 
+          <attribute name="X-Compile-Source-JDK"
                      value="${javac.source}"/>
-          <attribute name="X-Compile-Target-JDK" 
+          <attribute name="X-Compile-Target-JDK"
                      value="${javac.target}"/>
           <manifest-attributes/>
         </manifest>
@@ -309,7 +356,7 @@
            anywhere.
        -->
       <sysproperty key="lucene.common.dir" file="${common.dir}" />
-      
+
       <!-- contrib/ant IndexTaskTest needs these two system properties -->
       <sysproperty key="docs.dir" file="src/test"/>
       <sysproperty key="index.dir" file="${build.dir}/test/index"/>
@@ -333,7 +380,7 @@
     <fail if="tests.failed">Tests failed!</fail>
     <!-- life would be easier if echo had an 'if' attribute like fail -->
     <delete file="${build.dir}/test/junitfailed.flag" />
-    
+
   </target>
 
     <!--
Index: E:/projects/lucene/trunk/src/test/org/apache/lucene/analysis/fast/TestFastAnalyzer.java
===================================================================
--- E:/projects/lucene/trunk/src/test/org/apache/lucene/analysis/fast/TestFastAnalyzer.java	(revision 0)
+++ E:/projects/lucene/trunk/src/test/org/apache/lucene/analysis/fast/TestFastAnalyzer.java	(revision 0)
@@ -0,0 +1,132 @@
+package org.apache.lucene.analysis.fast;
+
+import java.io.StringReader;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.*;
+
+/**
+ * Copyright 2004 The Apache Software Foundation <p/> Licensed under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0 <p/> Unless required by applicable law or
+ * agreed to in writing, software distributed under the License is distributed on an "AS
+ * IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+ * the License for the specific language governing permissions and limitations under the
+ * License.
+ */
+
+public class TestFastAnalyzer extends TestCase {
+
+    private Analyzer a = new FastAnalyzer();
+
+    public void assertAnalyzesTo(Analyzer a, String input, String[] expected) throws Exception {
+	TokenStream ts = a.tokenStream("dummy", new StringReader(input));
+	for (int i = 0; i < expected.length; i++) {
+	    Token t = ts.next();
+	    assertNotNull(t);
+	    assertEquals(expected[i], t.termText());
+	}
+	assertNull(ts.next());
+	ts.close();
+    }
+
+
+    public void testAlphanumeric() throws Exception {
+	// alphanumeric tokens
+	assertAnalyzesTo(a, "B2B", new String[]{"b2b"});
+	assertAnalyzesTo(a, "2B", new String[]{"2b"});
+    }
+
+    public void testUnderscores() throws Exception {
+	// underscores are delimiters, but not in email addresses (below)
+	assertAnalyzesTo(a, "word_having_underscore", new String[]{"word", "having", "underscore"});
+	assertAnalyzesTo(a, "word_with_underscore_and_stopwords", new String[]{"word", "underscore", "stopwords"});
+    }
+
+    public void testDelimiters() throws Exception {
+	// other delimiters: "-", "/", ","
+	assertAnalyzesTo(a, "some-dashed-phrase",   new String[]{"some", "dashed", "phrase" });
+	assertAnalyzesTo(a, "dogs,chase,cats", new String[]{"dogs", "chase", "cats"});
+	assertAnalyzesTo(a, "ac/dc", new String[]{"ac", "dc"});
+    }
+
+    public void testApostrophes() throws Exception {
+	// internal apostrophes: O'Reilly, you're, O'Reilly's
+	// possessives are actually removed by StardardFilter, not the tokenizer
+	assertAnalyzesTo(a, "O'Reilly", new String[]{"o'reilly"});
+	assertAnalyzesTo(a, "you're", new String[]{"you're"});
+	assertAnalyzesTo(a, "she's", new String[]{"she"});
+	assertAnalyzesTo(a, "Jim's", new String[]{"jim"});
+	assertAnalyzesTo(a, "don't", new String[]{"don't"});
+	assertAnalyzesTo(a, "O'Reilly's", new String[]{"o'reilly"});
+    }
+
+    public void testTSADash() throws Exception {
+	// t and s had been stopwords in Lucene <= 2.0, which made it impossible
+	// to correctly search for these terms:
+	assertAnalyzesTo(a, "s-class", new String[]{"s", "class"});
+	assertAnalyzesTo(a, "t-com", new String[]{"t", "com"});
+	// 'a' is still a stopword:
+	assertAnalyzesTo(a, "a-class", new String[]{"class"});
+    }
+
+    public void testCompanyNames() throws Exception {
+	// company names
+	assertAnalyzesTo(a, "AT&T", new String[]{"at&t"});
+	assertAnalyzesTo(a, "Excite@Home", new String[]{"excite@home"});
+    }
+
+    public void testDomainNames() throws Exception {
+	// domain names
+	assertAnalyzesTo(a, "www.nutch.org",   new String[]{"www.nutch.org" });
+    }
+
+    public void testEmail() throws Exception {
+	// email addresses, possibly with underscores, periods, etc
+	assertAnalyzesTo(a, "test@example.com", new String[]{"test@example.com"});
+	assertAnalyzesTo(a, "first.lastname@example.com", new String[]{"first.lastname@example.com"});
+	assertAnalyzesTo(a, "first_lastname@example.com", new String[]{"first_lastname@example.com"});
+    }
+
+    public void testNumbers() throws Exception {
+	// floating point, serial, model numbers, ip addresses, etc.
+	// every other segment must have at least one digit
+	assertAnalyzesTo(a, "21.35", new String[]{"21.35"});
+	assertAnalyzesTo(a, "R2D2 C3PO", new String[]{"r2d2", "c3po"});
+	assertAnalyzesTo(a, "216.239.63.104",   new String[]{"216.239.63.104"});
+	assertAnalyzesTo(a, "1-2-3",   new String[]{"1-2-3"});
+	assertAnalyzesTo(a, "a1-b2-c3",   new String[]{"a1-b2-c3"});
+	assertAnalyzesTo(a, "a1-b-c3",   new String[]{"a1-b-c3"});
+    }
+
+    public void testWhitespace() throws Exception {
+	// numbers
+	assertAnalyzesTo(a, "David has 5000 bones", new String[]{"david", "has", "5000", "bones"});
+	assertAnalyzesTo(a, "C embedded developers wanted", new String[]{"c", "embedded", "developers", "wanted" });
+	assertAnalyzesTo(a, "foo bar FOO BAR", new String[]{"foo", "bar", "foo", "bar"});
+    }
+
+    public void testSpecialCharacters() throws Exception {
+	assertAnalyzesTo(a, "foo      bar .  FOO <> BAR", new String[]{"foo", "bar", "foo", "bar"});
+	assertAnalyzesTo(a, "\"QUOTED\" word", new String[]{"quoted", "word"});
+    }
+
+    public void testAcronyms() throws Exception {
+	// acronyms have their dots stripped
+	assertAnalyzesTo(a, "U.S.A.", new String[]{ "usa" });
+    }
+
+    public void testCPlusPlus() throws Exception {
+	// It would be nice to change the grammar in StandardTokenizer.jj to make "C#" and
+	// "C++" end up as tokens.
+	assertAnalyzesTo(a, "C++", new String[]{"c"});
+	assertAnalyzesTo(a, "C#", new String[]{"c"});
+    }
+
+    public void testKorean() throws Exception {
+	// Korean words
+	assertAnalyzesTo(a, "안녕하세요 한글입니다", new String[]{"안녕하세요", "한글입니다"});
+    }
+}
Index: E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizerImpl.jflex
===================================================================
--- E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizerImpl.jflex	(revision 0)
+++ E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizerImpl.jflex	(revision 0)
@@ -0,0 +1,104 @@
+package org.apache.lucene.analysis.fast;
+%%
+
+%class FastTokenizerImpl
+%unicode
+%integer
+%function getNextToken
+%pack
+%char
+
+%{
+
+public static final int ALPHANUM          = 0;
+public static final int APOSTROPHE        = 1;
+public static final int ACRONYM           = 2;
+public static final int COMPANY           = 3;
+public static final int EMAIL             = 4;
+public static final int HOST              = 5;
+public static final int NUM               = 6;
+public static final int CJ                = 7;
+
+public static final String [] TOKEN_TYPES = new String [] {
+    "<ALPHANUM>",
+    "<APOSTROPHE>",
+    "<ACRONYM>",
+    "<COMPANY>",
+    "<EMAIL>",
+    "<HOST>",
+    "<NUM>",
+    "<CJ>"
+};
+
+public final int yychar()
+{
+    return yychar;
+}
+%}
+
+// basic word: a sequence of digits & letters
+ALPHANUM   = ({LETTER}|{DIGIT}|{KOREAN})+
+
+// internal apostrophes: O'Reilly, you're, O'Reilly's
+// use a post-filter to remove possesives
+APOSTROPHE =  {ALPHA} ("'" {ALPHA})+
+
+// acronyms: U.S.A., I.B.M., etc.
+// use a post-filter to remove dots
+ACRONYM    =  {ALPHA} "." ({ALPHA} ".")+
+
+// company names like AT&T and Excite@Home.
+COMPANY    =  {ALPHA} ("&"|"@") {ALPHA}
+
+// email addresses
+EMAIL      =  {ALPHANUM} (("."|"-"|"_") {ALPHANUM})* "@" {ALPHANUM} (("."|"-") {ALPHANUM})+
+
+// hostname
+HOST       =  {ALPHANUM} ("." {ALPHANUM})+
+
+// floating point, serial, model numbers, ip addresses, etc.
+// every other segment must have at least one digit
+NUM        = ({ALPHANUM} {P} {HAS_DIGIT}
+       | {HAS_DIGIT} {P} {ALPHANUM}
+       | {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+
+       | {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
+       | {ALPHANUM} {P} {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
+       | {HAS_DIGIT} {P} {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+
+        )
+
+// punctuation
+P	         = ("_"|"-"|"/"|"."|",")
+
+// at least one digit
+HAS_DIGIT  =
+    ({LETTER}|{DIGIT})*
+    {DIGIT}
+    ({LETTER}|{DIGIT})*
+
+ALPHA      = ({LETTER})+
+
+
+LETTER     = [\u0041-\u005a\u0061-\u007a\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u00ff\u0100-\u1fff\uffa0-\uffdc]
+
+DIGIT      = [\u0030-\u0039\u0660-\u0669\u06f0-\u06f9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be7-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0e50-\u0e59\u0ed0-\u0ed9\u1040-\u1049]
+
+KOREAN     = [\uac00-\ud7af\u1100-\u11ff]
+
+// Chinese, Japanese
+CJ         = [\u3040-\u318f\u3100-\u312f\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3300-\u337f\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff65-\uff9f]
+
+WHITESPACE = \r\n | [ \r\n\t\f]
+
+%%
+
+{ALPHANUM}                                                     { return ALPHANUM; }
+{APOSTROPHE}                                                   { return APOSTROPHE; }
+{ACRONYM}                                                      { return ACRONYM; }
+{COMPANY}                                                      { return COMPANY; }
+{EMAIL}                                                        { return EMAIL; }
+{HOST}                                                         { return HOST; }
+{NUM}                                                          { return NUM; }
+{CJ}                                                           { return CJ; }
+
+/** Ignore the rest */
+. | {WHITESPACE}                                               { /* ignore */ }
Index: E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastAnalyzer.java
===================================================================
--- E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastAnalyzer.java	(revision 0)
+++ E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastAnalyzer.java	(revision 0)
@@ -0,0 +1,78 @@
+package org.apache.lucene.analysis.fast;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.*;
+import java.util.Set;
+
+import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+
+/**
+ * Filters {@link FastTokenizer} with {@link StandardFilter}, {@link
+ * LowerCaseFilter} and {@link StopFilter}, using a list of English stop words.
+ *
+ * @version $Id: StandardAnalyzer.java 472959 2006-11-09 16:21:50Z yonik $
+ */
+public class FastAnalyzer extends Analyzer {
+  private Set stopSet;
+
+  /** An array containing some common English words that are usually not
+  useful for searching. */
+  public static final String[] STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS;
+
+  /** Builds an analyzer with the default stop words ({@link #STOP_WORDS}). */
+  public FastAnalyzer() {
+    this(STOP_WORDS);
+  }
+
+  /** Builds an analyzer with the given stop words. */
+  public FastAnalyzer(Set stopWords) {
+    stopSet = stopWords;
+  }
+
+  /** Builds an analyzer with the given stop words. */
+  public FastAnalyzer(String[] stopWords) {
+    stopSet = StopFilter.makeStopSet(stopWords);
+  }
+
+  /** Builds an analyzer with the stop words from the given file.
+   * @see WordlistLoader#getWordSet(File)
+   */
+  public FastAnalyzer(File stopwords) throws IOException {
+    stopSet = WordlistLoader.getWordSet(stopwords);
+  }
+
+  /** Builds an analyzer with the stop words from the given reader.
+   * @see WordlistLoader#getWordSet(Reader)
+   */
+  public FastAnalyzer(Reader stopwords) throws IOException {
+    stopSet = WordlistLoader.getWordSet(stopwords);
+  }
+
+  /** Constructs a {@link StandardTokenizer} filtered by a {@link
+  StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */
+  public TokenStream tokenStream(String fieldName, Reader reader) {
+    TokenStream result = new FastTokenizer(reader);
+    result = new StandardFilter(result);
+    result = new LowerCaseFilter(result);
+    result = new StopFilter(result, stopSet);
+    return result;
+  }
+}
Index: E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastFilter.java
===================================================================
--- E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastFilter.java	(revision 0)
+++ E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastFilter.java	(revision 0)
@@ -0,0 +1,68 @@
+package org.apache.lucene.analysis.fast;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.*;
+
+/** Normalizes tokens extracted with {@link FastTokenizer}. */
+
+public final class FastFilter extends TokenFilter {
+
+
+  /** Construct filtering <i>in</i>. */
+  public FastFilter(TokenStream in) {
+    super(in);
+  }
+
+  private static final String APOSTROPHE_TYPE = FastTokenizerImpl.TOKEN_TYPES[FastTokenizerImpl.APOSTROPHE];
+  private static final String ACRONYM_TYPE = FastTokenizerImpl.TOKEN_TYPES[FastTokenizerImpl.ACRONYM];
+
+  /** Returns the next token in the stream, or null at EOS.
+   * <p>Removes <tt>'s</tt> from the end of words.
+   * <p>Removes dots from acronyms.
+   */
+  public final org.apache.lucene.analysis.Token next() throws java.io.IOException {
+    org.apache.lucene.analysis.Token t = input.next();
+
+    if (t == null)
+      return null;
+
+    String text = t.termText();
+    String type = t.type();
+
+    if (type == APOSTROPHE_TYPE &&		  // remove 's
+	(text.endsWith("'s") || text.endsWith("'S"))) {
+      return new org.apache.lucene.analysis.Token
+	(text.substring(0,text.length()-2),
+	 t.startOffset(), t.endOffset(), type);
+
+    } else if (type == ACRONYM_TYPE) {		  // remove dots
+      StringBuffer trimmed = new StringBuffer();
+      for (int i = 0; i < text.length(); i++) {
+	char c = text.charAt(i);
+	if (c != '.')
+	  trimmed.append(c);
+      }
+      return new org.apache.lucene.analysis.Token
+	(trimmed.toString(), t.startOffset(), t.endOffset(), type);
+
+    } else {
+      return t;
+    }
+  }
+}
Index: E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizer.java
===================================================================
--- E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizer.java	(revision 0)
+++ E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizer.java	(revision 0)
@@ -0,0 +1,57 @@
+/*
+ * Carrot2 project.
+ *
+ * Copyright (C) 2002-2007, Dawid Weiss, Stanisław Osiński.
+ * Portions (C) Contributors listed in "carrot2.CONTRIBUTORS" file.
+ * All rights reserved.
+ *
+ * Refer to the full license file "carrot2.LICENSE"
+ * in the root folder of the repository checkout or at:
+ * http://www.carrot2.org/carrot2.LICENSE
+ */
+
+package org.apache.lucene.analysis.fast;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.Tokenizer;
+
+/**
+ * An interface to the tokenizer generated by JFlex.
+ *
+ * @author Stanislaw Osinski
+ */
+public class FastTokenizer extends Tokenizer
+{
+    private final FastTokenizerImpl scanner;
+
+    public FastTokenizer(Reader input)
+    {
+        this.input = input;
+        this.scanner = new FastTokenizerImpl(input);
+    }
+
+    public Token next() throws IOException
+    {
+        int tokenType = scanner.getNextToken();
+
+        if (tokenType == FastTokenizerImpl.YYEOF)
+        {
+            return null;
+        }
+
+        int startPosition = scanner.yychar();
+
+        final String tokenImage = scanner.yytext();
+        return new Token(tokenImage, startPosition, startPosition + tokenImage.length(),
+            FastTokenizerImpl.TOKEN_TYPES[tokenType]);
+    }
+
+    public void reset() throws IOException
+    {
+        super.reset();
+        scanner.yyreset(input);
+    }
+}
Index: E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizerImpl.java
===================================================================
--- E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizerImpl.java	(revision 0)
+++ E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizerImpl.java	(revision 0)
@@ -0,0 +1,627 @@
+/* The following code was generated by JFlex 1.4.1 on 07-07-26 11:38 */
+
+package org.apache.lucene.analysis.fast;
+
+/**
+ * This class is a scanner generated by 
+ * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
+ * on 07-07-26 11:38 from the specification file
+ * <tt>E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/FastTokenizerImpl.jflex</tt>
+ */
+class FastTokenizerImpl {
+
+  /** This character denotes the end of file */
+  public static final int YYEOF = -1;
+
+  /** initial size of the lookahead buffer */
+  private static final int ZZ_BUFFERSIZE = 16384;
+
+  /** lexical states */
+  public static final int YYINITIAL = 0;
+
+  /** 
+   * Translates characters to character classes
+   */
+  private static final String ZZ_CMAP_PACKED = 
+    "\11\0\1\0\1\16\1\0\1\0\1\15\22\0\1\0\5\0\1\3"+
+    "\1\1\4\0\1\7\1\5\1\2\1\7\12\11\6\0\1\4\32\10"+
+    "\4\0\1\6\1\0\32\10\105\0\27\10\1\0\37\10\1\0\u0568\10"+
+    "\12\12\206\10\12\12\u026c\10\12\12\166\10\12\12\166\10\12\12\166\10"+
+    "\12\12\166\10\12\12\167\10\11\12\166\10\12\12\166\10\12\12\166\10"+
+    "\12\12\340\10\12\12\166\10\12\12\u0166\10\12\12\266\10\u0100\10\u0e00\10"+
+    "\u1040\0\u0150\14\140\0\20\14\u0100\0\200\14\200\0\u19c0\14\100\0\u5200\14"+
+    "\u0c00\0\u2bb0\13\u2150\0\u0200\14\u0465\0\73\14\75\10\43\0";
+
+  /** 
+   * Translates characters to character classes
+   */
+  private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
+
+  /** 
+   * Translates DFA states to action switch labels.
+   */
+  private static final int [] ZZ_ACTION = zzUnpackAction();
+
+  private static final String ZZ_ACTION_PACKED_0 =
+    "\1\0\1\1\4\2\1\3\1\1\14\0\1\4\4\5"+
+    "\2\6\2\0\1\7\1\0\1\7\3\5\6\7\3\5"+
+    "\1\10\4\0\1\10\2\0\2\10\2\5\1\11";
+
+  private static int [] zzUnpackAction() {
+    int [] result = new int[57];
+    int offset = 0;
+    offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
+    return result;
+  }
+
+  private static int zzUnpackAction(String packed, int offset, int [] result) {
+    int i = 0;       /* index in packed string  */
+    int j = offset;  /* index in unpacked array */
+    int l = packed.length();
+    while (i < l) {
+      int count = packed.charAt(i++);
+      int value = packed.charAt(i++);
+      do result[j++] = value; while (--count > 0);
+    }
+    return j;
+  }
+
+
+  /** 
+   * Translates a state to a row index in the transition table
+   */
+  private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
+
+  private static final String ZZ_ROWMAP_PACKED_0 =
+    "\0\0\0\17\0\36\0\55\0\74\0\113\0\17\0\132"+
+    "\0\151\0\170\0\207\0\226\0\245\0\264\0\303\0\322"+
+    "\0\341\0\360\0\377\0\u010e\0\u011d\0\u012c\0\u013b\0\u014a"+
+    "\0\u0159\0\207\0\u0168\0\u0177\0\u0186\0\u0195\0\u01a4\0\u01b3"+
+    "\0\u01c2\0\u01d1\0\u01e0\0\u01ef\0\u01fe\0\u020d\0\u021c\0\u022b"+
+    "\0\u023a\0\u0249\0\u0258\0\u0267\0\u0276\0\u0285\0\u0294\0\u02a3"+
+    "\0\u02b2\0\u02c1\0\u02d0\0\u02df\0\170\0\377\0\u02ee\0\u02fd"+
+    "\0\u030c";
+
+  private static int [] zzUnpackRowMap() {
+    int [] result = new int[57];
+    int offset = 0;
+    offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
+    return result;
+  }
+
+  private static int zzUnpackRowMap(String packed, int offset, int [] result) {
+    int i = 0;  /* index in packed string  */
+    int j = offset;  /* index in unpacked array */
+    int l = packed.length();
+    while (i < l) {
+      int high = packed.charAt(i++) << 16;
+      result[j++] = high | packed.charAt(i++);
+    }
+    return j;
+  }
+
+  /** 
+   * The transition table of the DFA
+   */
+  private static final int [] ZZ_TRANS = zzUnpackTrans();
+
+  private static final String ZZ_TRANS_PACKED_0 =
+    "\10\2\1\3\1\4\1\5\1\6\1\7\1\10\1\2"+
+    "\20\0\1\11\1\12\1\13\1\14\2\15\1\16\1\3"+
+    "\1\4\1\5\1\6\5\0\1\17\1\0\1\20\2\21"+
+    "\1\22\3\4\1\6\4\0\1\11\1\23\1\13\1\14"+
+    "\2\21\1\22\1\5\1\4\1\5\1\6\5\0\1\24"+
+    "\1\0\1\20\2\15\1\16\4\6\21\0\1\2\10\0"+
+    "\1\25\1\0\1\25\14\0\1\26\1\27\1\30\1\31"+
+    "\13\0\1\32\1\0\1\32\14\0\1\33\1\34\1\33"+
+    "\1\34\13\0\1\35\2\36\1\37\13\0\1\16\2\40"+
+    "\14\0\1\41\2\42\1\43\13\0\4\34\13\0\1\44"+
+    "\2\45\1\46\13\0\1\47\2\50\1\51\13\0\1\52"+
+    "\1\42\1\53\1\43\13\0\1\54\2\27\1\31\4\0"+
+    "\1\11\6\0\1\25\1\0\1\25\6\0\1\55\1\0"+
+    "\1\20\2\56\1\0\1\26\1\27\1\30\1\31\5\0"+
+    "\1\57\1\0\1\20\2\60\1\61\3\27\1\31\5\0"+
+    "\1\62\1\0\1\20\2\60\1\61\1\30\1\27\1\30"+
+    "\1\31\5\0\1\63\1\0\1\20\2\56\1\0\4\31"+
+    "\5\0\1\64\2\0\1\64\2\0\1\33\1\34\1\33"+
+    "\1\34\5\0\1\64\2\0\1\64\2\0\4\34\5\0"+
+    "\1\56\1\0\1\20\2\56\1\0\1\35\2\36\1\37"+
+    "\5\0\1\60\1\0\1\20\2\60\1\61\3\36\1\37"+
+    "\5\0\1\56\1\0\1\20\2\56\1\0\4\37\5\0"+
+    "\1\61\2\0\3\61\3\40\6\0\1\24\1\0\1\20"+
+    "\2\15\1\16\1\41\2\42\1\43\5\0\1\17\1\0"+
+    "\1\20\2\21\1\22\3\42\1\43\5\0\1\24\1\0"+
+    "\1\20\2\15\1\16\4\43\5\0\1\15\1\0\1\20"+
+    "\2\15\1\16\1\44\2\45\1\46\5\0\1\21\1\0"+
+    "\1\20\2\21\1\22\3\45\1\46\5\0\1\15\1\0"+
+    "\1\20\2\15\1\16\4\46\5\0\1\16\2\0\3\16"+
+    "\1\47\2\50\1\51\5\0\1\22\2\0\3\22\3\50"+
+    "\1\51\5\0\1\16\2\0\3\16\4\51\5\0\1\65"+
+    "\1\0\1\20\2\15\1\16\1\52\1\42\1\53\1\43"+
+    "\5\0\1\66\1\0\1\20\2\21\1\22\1\53\1\42"+
+    "\1\53\1\43\5\0\1\63\1\0\1\20\2\56\1\0"+
+    "\1\54\2\27\1\31\13\0\1\67\1\31\1\67\1\31"+
+    "\13\0\4\37\13\0\4\43\13\0\4\46\13\0\4\51"+
+    "\13\0\1\70\1\43\1\70\1\43\13\0\4\31\13\0"+
+    "\4\71\5\0\1\55\1\0\1\20\2\56\1\0\1\67"+
+    "\1\31\1\67\1\31\5\0\1\65\1\0\1\20\2\15"+
+    "\1\16\1\70\1\43\1\70\1\43\5\0\1\64\2\0"+
+    "\1\64\2\0\4\71\3\0";
+
+  private static int [] zzUnpackTrans() {
+    int [] result = new int[795];
+    int offset = 0;
+    offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
+    return result;
+  }
+
+  private static int zzUnpackTrans(String packed, int offset, int [] result) {
+    int i = 0;       /* index in packed string  */
+    int j = offset;  /* index in unpacked array */
+    int l = packed.length();
+    while (i < l) {
+      int count = packed.charAt(i++);
+      int value = packed.charAt(i++);
+      value--;
+      do result[j++] = value; while (--count > 0);
+    }
+    return j;
+  }
+
+
+  /* error codes */
+  private static final int ZZ_UNKNOWN_ERROR = 0;
+  private static final int ZZ_NO_MATCH = 1;
+  private static final int ZZ_PUSHBACK_2BIG = 2;
+
+  /* error messages for the codes above */
+  private static final String ZZ_ERROR_MSG[] = {
+    "Unkown internal scanner error",
+    "Error: could not match input",
+    "Error: pushback value was too large"
+  };
+
+  /**
+   * ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
+   */
+  private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
+
+  private static final String ZZ_ATTRIBUTE_PACKED_0 =
+    "\1\0\1\11\4\1\1\11\1\1\14\0\7\1\2\0"+
+    "\1\1\1\0\16\1\4\0\1\1\2\0\5\1";
+
+  private static int [] zzUnpackAttribute() {
+    int [] result = new int[57];
+    int offset = 0;
+    offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
+    return result;
+  }
+
+  private static int zzUnpackAttribute(String packed, int offset, int [] result) {
+    int i = 0;       /* index in packed string  */
+    int j = offset;  /* index in unpacked array */
+    int l = packed.length();
+    while (i < l) {
+      int count = packed.charAt(i++);
+      int value = packed.charAt(i++);
+      do result[j++] = value; while (--count > 0);
+    }
+    return j;
+  }
+
+  /** the input device */
+  private java.io.Reader zzReader;
+
+  /** the current state of the DFA */
+  private int zzState;
+
+  /** the current lexical state */
+  private int zzLexicalState = YYINITIAL;
+
+  /** this buffer contains the current text to be matched and is
+      the source of the yytext() string */
+  private char zzBuffer[] = new char[ZZ_BUFFERSIZE];
+
+  /** the textposition at the last accepting state */
+  private int zzMarkedPos;
+
+  /** the textposition at the last state to be included in yytext */
+  private int zzPushbackPos;
+
+  /** the current text position in the buffer */
+  private int zzCurrentPos;
+
+  /** startRead marks the beginning of the yytext() string in the buffer */
+  private int zzStartRead;
+
+  /** endRead marks the last character in the buffer, that has been read
+      from input */
+  private int zzEndRead;
+
+  /** number of newlines encountered up to the start of the matched text */
+  private int yyline;
+
+  /** the number of characters up to the start of the matched text */
+  private int yychar;
+
+  /**
+   * the number of characters from the last newline up to the start of the 
+   * matched text
+   */
+  private int yycolumn;
+
+  /** 
+   * zzAtBOL == true <=> the scanner is currently at the beginning of a line
+   */
+  private boolean zzAtBOL = true;
+
+  /** zzAtEOF == true <=> the scanner is at the EOF */
+  private boolean zzAtEOF;
+
+  /* user code: */
+
+public static final int ALPHANUM          = 0;
+public static final int APOSTROPHE        = 1;
+public static final int ACRONYM           = 2;
+public static final int COMPANY           = 3;
+public static final int EMAIL             = 4;
+public static final int HOST              = 5;
+public static final int NUM               = 6;
+public static final int CJ                = 7;
+
+public static final String [] TOKEN_TYPES = new String [] {
+    "<ALPHANUM>",
+    "<APOSTROPHE>",
+    "<ACRONYM>",
+    "<COMPANY>",
+    "<EMAIL>",
+    "<HOST>",
+    "<NUM>",
+    "<CJ>"
+};
+
+public final int yychar()
+{
+    return yychar;
+}
+
+
+  /**
+   * Creates a new scanner
+   * There is also a java.io.InputStream version of this constructor.
+   *
+   * @param   in  the java.io.Reader to read input from.
+   */
+  FastTokenizerImpl(java.io.Reader in) {
+    this.zzReader = in;
+  }
+
+  /**
+   * Creates a new scanner.
+   * There is also java.io.Reader version of this constructor.
+   *
+   * @param   in  the java.io.Inputstream to read input from.
+   */
+  FastTokenizerImpl(java.io.InputStream in) {
+    this(new java.io.InputStreamReader(in));
+  }
+
+  /** 
+   * Unpacks the compressed character translation table.
+   *
+   * @param packed   the packed character translation table
+   * @return         the unpacked character translation table
+   */
+  private static char [] zzUnpackCMap(String packed) {
+    char [] map = new char[0x10000];
+    int i = 0;  /* index in packed string  */
+    int j = 0;  /* index in unpacked array */
+    while (i < 156) {
+      int  count = packed.charAt(i++);
+      char value = packed.charAt(i++);
+      do map[j++] = value; while (--count > 0);
+    }
+    return map;
+  }
+
+
+  /**
+   * Refills the input buffer.
+   *
+   * @return      <code>false</code>, iff there was new input.
+   * 
+   * @exception   java.io.IOException  if any I/O-Error occurs
+   */
+  private boolean zzRefill() throws java.io.IOException {
+
+    /* first: make room (if you can) */
+    if (zzStartRead > 0) {
+      System.arraycopy(zzBuffer, zzStartRead,
+                       zzBuffer, 0,
+                       zzEndRead-zzStartRead);
+
+      /* translate stored positions */
+      zzEndRead-= zzStartRead;
+      zzCurrentPos-= zzStartRead;
+      zzMarkedPos-= zzStartRead;
+      zzPushbackPos-= zzStartRead;
+      zzStartRead = 0;
+    }
+
+    /* is the buffer big enough? */
+    if (zzCurrentPos >= zzBuffer.length) {
+      /* if not: blow it up */
+      char newBuffer[] = new char[zzCurrentPos*2];
+      System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length);
+      zzBuffer = newBuffer;
+    }
+
+    /* finally: fill the buffer with new input */
+    int numRead = zzReader.read(zzBuffer, zzEndRead,
+                                            zzBuffer.length-zzEndRead);
+
+    if (numRead < 0) {
+      return true;
+    }
+    else {
+      zzEndRead+= numRead;
+      return false;
+    }
+  }
+
+    
+  /**
+   * Closes the input stream.
+   */
+  public final void yyclose() throws java.io.IOException {
+    zzAtEOF = true;            /* indicate end of file */
+    zzEndRead = zzStartRead;  /* invalidate buffer    */
+
+    if (zzReader != null)
+      zzReader.close();
+  }
+
+
+  /**
+   * Resets the scanner to read from a new input stream.
+   * Does not close the old reader.
+   *
+   * All internal variables are reset, the old input stream 
+   * <b>cannot</b> be reused (internal buffer is discarded and lost).
+   * Lexical state is set to <tt>ZZ_INITIAL</tt>.
+   *
+   * @param reader   the new input stream 
+   */
+  public final void yyreset(java.io.Reader reader) {
+    zzReader = reader;
+    zzAtBOL  = true;
+    zzAtEOF  = false;
+    zzEndRead = zzStartRead = 0;
+    zzCurrentPos = zzMarkedPos = zzPushbackPos = 0;
+    yyline = yychar = yycolumn = 0;
+    zzLexicalState = YYINITIAL;
+  }
+
+
+  /**
+   * Returns the current lexical state.
+   */
+  public final int yystate() {
+    return zzLexicalState;
+  }
+
+
+  /**
+   * Enters a new lexical state
+   *
+   * @param newState the new lexical state
+   */
+  public final void yybegin(int newState) {
+    zzLexicalState = newState;
+  }
+
+
+  /**
+   * Returns the text matched by the current regular expression.
+   */
+  public final String yytext() {
+    return new String( zzBuffer, zzStartRead, zzMarkedPos-zzStartRead );
+  }
+
+
+  /**
+   * Returns the character at position <tt>pos</tt> from the 
+   * matched text. 
+   * 
+   * It is equivalent to yytext().charAt(pos), but faster
+   *
+   * @param pos the position of the character to fetch. 
+   *            A value from 0 to yylength()-1.
+   *
+   * @return the character at position pos
+   */
+  public final char yycharat(int pos) {
+    return zzBuffer[zzStartRead+pos];
+  }
+
+
+  /**
+   * Returns the length of the matched text region.
+   */
+  public final int yylength() {
+    return zzMarkedPos-zzStartRead;
+  }
+
+
+  /**
+   * Reports an error that occured while scanning.
+   *
+   * In a wellformed scanner (no or only correct usage of 
+   * yypushback(int) and a match-all fallback rule) this method 
+   * will only be called with things that "Can't Possibly Happen".
+   * If this method is called, something is seriously wrong
+   * (e.g. a JFlex bug producing a faulty scanner etc.).
+   *
+   * Usual syntax/scanner level error handling should be done
+   * in error fallback rules.
+   *
+   * @param   errorCode  the code of the errormessage to display
+   */
+  private void zzScanError(int errorCode) {
+    String message;
+    try {
+      message = ZZ_ERROR_MSG[errorCode];
+    }
+    catch (ArrayIndexOutOfBoundsException e) {
+      message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
+    }
+
+    throw new Error(message);
+  } 
+
+
+  /**
+   * Pushes the specified amount of characters back into the input stream.
+   *
+   * They will be read again by then next call of the scanning method
+   *
+   * @param number  the number of characters to be read again.
+   *                This number must not be greater than yylength()!
+   */
+  public void yypushback(int number)  {
+    if ( number > yylength() )
+      zzScanError(ZZ_PUSHBACK_2BIG);
+
+    zzMarkedPos -= number;
+  }
+
+
+  /**
+   * Resumes scanning until the next regular expression is matched,
+   * the end of input is encountered or an I/O-Error occurs.
+   *
+   * @return      the next token
+   * @exception   java.io.IOException  if any I/O-Error occurs
+   */
+  public int getNextToken() throws java.io.IOException {
+    int zzInput;
+    int zzAction;
+
+    // cached fields:
+    int zzCurrentPosL;
+    int zzMarkedPosL;
+    int zzEndReadL = zzEndRead;
+    char [] zzBufferL = zzBuffer;
+    char [] zzCMapL = ZZ_CMAP;
+
+    int [] zzTransL = ZZ_TRANS;
+    int [] zzRowMapL = ZZ_ROWMAP;
+    int [] zzAttrL = ZZ_ATTRIBUTE;
+
+    while (true) {
+      zzMarkedPosL = zzMarkedPos;
+
+      yychar+= zzMarkedPosL-zzStartRead;
+
+      zzAction = -1;
+
+      zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
+  
+      zzState = zzLexicalState;
+
+
+      zzForAction: {
+        while (true) {
+    
+          if (zzCurrentPosL < zzEndReadL)
+            zzInput = zzBufferL[zzCurrentPosL++];
+          else if (zzAtEOF) {
+            zzInput = YYEOF;
+            break zzForAction;
+          }
+          else {
+            // store back cached positions
+            zzCurrentPos  = zzCurrentPosL;
+            zzMarkedPos   = zzMarkedPosL;
+            boolean eof = zzRefill();
+            // get translated positions and possibly new buffer
+            zzCurrentPosL  = zzCurrentPos;
+            zzMarkedPosL   = zzMarkedPos;
+            zzBufferL      = zzBuffer;
+            zzEndReadL     = zzEndRead;
+            if (eof) {
+              zzInput = YYEOF;
+              break zzForAction;
+            }
+            else {
+              zzInput = zzBufferL[zzCurrentPosL++];
+            }
+          }
+          int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
+          if (zzNext == -1) break zzForAction;
+          zzState = zzNext;
+
+          int zzAttributes = zzAttrL[zzState];
+          if ( (zzAttributes & 1) == 1 ) {
+            zzAction = zzState;
+            zzMarkedPosL = zzCurrentPosL;
+            if ( (zzAttributes & 8) == 8 ) break zzForAction;
+          }
+
+        }
+      }
+
+      // store back cached position
+      zzMarkedPos = zzMarkedPosL;
+
+      switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
+        case 5: 
+          { return HOST;
+          }
+        case 10: break;
+        case 8: 
+          { return ACRONYM;
+          }
+        case 11: break;
+        case 1: 
+          { /* ignore */
+          }
+        case 12: break;
+        case 7: 
+          { return NUM;
+          }
+        case 13: break;
+        case 3: 
+          { return CJ;
+          }
+        case 14: break;
+        case 2: 
+          { return ALPHANUM;
+          }
+        case 15: break;
+        case 6: 
+          { return COMPANY;
+          }
+        case 16: break;
+        case 4: 
+          { return APOSTROPHE;
+          }
+        case 17: break;
+        case 9: 
+          { return EMAIL;
+          }
+        case 18: break;
+        default: 
+          if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
+            zzAtEOF = true;
+            return YYEOF;
+          } 
+          else {
+            zzScanError(ZZ_NO_MATCH);
+          }
+      }
+    }
+  }
+
+
+}
Index: E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/package.html
===================================================================
--- E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/package.html	(revision 0)
+++ E:/projects/lucene/trunk/src/java/org/apache/lucene/analysis/fast/package.html	(revision 0)
@@ -0,0 +1,10 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+   <meta name="Author" content="Stanislaw Osinski">
+</head>
+<body>
+A fast grammar-based tokenizer constructed with JFlex.
+</body>
+</html>
Index: E:/projects/lucene/trunk/build.xml
===================================================================
--- E:/projects/lucene/trunk/build.xml	(revision 559784)
+++ E:/projects/lucene/trunk/build.xml	(working copy)
@@ -7,9 +7,9 @@
     The ASF licenses this file to You under the Apache License, Version 2.0
     the "License"); you may not use this file except in compliance with
     the License.  You may obtain a copy of the License at
- 
+
         http://www.apache.org/licenses/LICENSE-2.0
- 
+
     Unless required by applicable law or agreed to in writing, software
     distributed under the License is distributed on an "AS IS" BASIS,
     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -23,7 +23,7 @@
 
   <import file="common-build.xml"/>
 
-  <property name="build.demo.template" value="src/demo/demo-build.template"/> 
+  <property name="build.demo.template" value="src/demo/demo-build.template"/>
 
   <property name="demo.name" value="lucene-demos-${version}"/>
   <property name="demo.war.name" value="luceneweb"/>
@@ -31,7 +31,7 @@
 
   <!-- Type of checksum to compute for distribution files -->
   <property name="checksum.algorithm" value="md5" />
-	
+
   <!-- Build classpath -->
   <path id="classpath">
     <pathelement location="${build.dir}/classes/java"/>
@@ -98,7 +98,7 @@
   <target name="test" depends="test-core, test-contrib"
           description="Runs all unit tests (including contribs)"
   />
-  
+
   <!-- ================================================================== -->
   <!-- J A R                                                              -->
   <!-- ================================================================== -->
@@ -122,11 +122,11 @@
                    value="${version}"/>
         <attribute name="Implementation-Vendor"
                    value="The Apache Software Foundation"/>
-        <attribute name="X-Compile-Source-JDK" 
+        <attribute name="X-Compile-Source-JDK"
                    value="${javac.source}"/>
-        <attribute name="X-Compile-Target-JDK" 
+        <attribute name="X-Compile-Target-JDK"
                    value="${javac.target}"/>
-	  </manifest>    	
+	  </manifest>
       <metainf dir="${common.dir}">
         <include name="LICENSE.txt"/>
         <include name="NOTICE.txt"/>
@@ -152,11 +152,11 @@
                    value="${version}"/>
         <attribute name="Implementation-Vendor"
                    value="The Apache Software Foundation"/>
-        <attribute name="X-Compile-Source-JDK" 
+        <attribute name="X-Compile-Source-JDK"
                    value="${javac.source}"/>
-        <attribute name="X-Compile-Target-JDK" 
+        <attribute name="X-Compile-Target-JDK"
                    value="${javac.target}"/>
-	  </manifest>    	
+	  </manifest>
       <metainf dir="${common.dir}">
         <include name="LICENSE.txt"/>
         <include name="NOTICE.txt"/>
@@ -224,7 +224,7 @@
   <!-- ================================================================== -->
   <target name="package" depends="jar-core, javadocs, war-demo, build-contrib, init-dist">
      <copy file="${build.demo.template}" tofile="${build.dir}/build-demo.xml">
-        <filterset begintoken="@PLACEHOLDER_" endtoken="@"> 
+        <filterset begintoken="@PLACEHOLDER_" endtoken="@">
 	  <filter token="version" value="${version}"/>
 	  <filter token="javac.source" value="${javac.source}"/>
 	  <filter token="javac.target" value="${javac.target}"/>
@@ -372,14 +372,14 @@
       <checksum forceOverwrite="yes" fileext=".md5">
         <fileset dir="${maven.dist.dir}" excludes="**/*.md5"/>
       </checksum>
-	
+
       <!-- do sha1 checksums -->
       <checksum forceOverwrite="yes" algorithm="SHA" fileext=".sha1">
         <fileset dir="${maven.dist.dir}" excludes="**/*.sha1, **/*.md5"/>
       </checksum>
     </sequential>
   </target>
-	
+
   <!-- ================================================================== -->
   <!-- Build the JavaCC files into the source tree                        -->
   <!-- ================================================================== -->
@@ -436,13 +436,37 @@
                    outputDir="src/java/org/apache/lucene/queryParser"
     />
   </target>
-  
+
   <target name="javacc-HTMLParser" depends="init,javacc-check" if="javacc.present">
     <invoke-javacc target="src/demo/org/apache/lucene/demo/html/HTMLParser.jj"
                    outputDir="src/demo/org/apache/lucene/demo/html"
     />
   </target>
-  
+
+  <!-- ================================================================== -->
+  <!-- Build the JFlex files into the source tree                         -->
+  <!-- ================================================================== -->
+
+  <target name="jflex" depends="clean-jflex,jflex-FastAnalyzer" />
+
+  <target name="jflex-FastAnalyzer" depends="init,jflex-check" if="jflex.present">
+    <taskdef classname="JFlex.anttask.JFlexTask" name="jflex">
+      <classpath location="${jflex.home}/lib/JFlex.jar" />
+    </taskdef>
+
+    <jflex file="src/java/org/apache/lucene/analysis/fast/FastTokenizerImpl.jflex"
+           outdir="src/java/org/apache/lucene/analysis/fast"
+           nobak="on" />
+  </target>
+
+  <target name="clean-jflex">
+    <delete>
+      <fileset dir="src/java/org/apache/lucene/analysis/fast" includes="*.java">
+        <containsregexp expression="generated.*by.*JFlex"/>
+      </fileset>
+    </delete>
+  </target>
+
   <macrodef name="contrib-crawl">
     <attribute name="target" default=""/>
     <attribute name="failonerror" default="true"/>
@@ -485,7 +509,7 @@
     <attribute name="access"/>
     <attribute name="destdir"/>
     <sequential>
-    
+
       <dirset dir="contrib/gdata-server/src/core/src/java" id="gdata-server-core">
         <include name="**" if="build-1-5-contrib" />
       </dirset>
@@ -520,7 +544,7 @@
 
         <!-- please keep this list up to date, and in alpha order...   -->
         <!-- with the minor exception of gdata which is managed by ref -->
-        
+
         <!-- ie: `find contrib/* -path \*src/java | grep -v gdata | sort` -->
 
         <!-- if you make changes to the list of package sets, also -->
@@ -552,7 +576,7 @@
         <!-- If the main javadoc Group listing includes an "Other   -->
         <!-- Packages" group after the ones listed here, then those -->
         <!-- packages are not being matched by any of these rules   -->
-        
+
         <group title="Core" packages="org.apache.*:org.apache.lucene.analysis:org.apache.lucene.analysis.standard*"/>
 
         <group title="Demo" packages="org.apache.lucene.demo*"/>
@@ -585,7 +609,7 @@
 
    </sequential>
   </macrodef>
-	
+
   <!-- Macro for building checksum files
        This is only needed until the "format" option is supported
        by ant's built in checksum task