Index: src/test/org/apache/lucene/index/TestFieldDataStore.java
===================================================================
RCS file: src/test/org/apache/lucene/index/TestFieldDataStore.java
diff -N src/test/org/apache/lucene/index/TestFieldDataStore.java
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ src/test/org/apache/lucene/index/TestFieldDataStore.java	1 Jan 1970 00:00:00 -0000
@@ -0,0 +1,257 @@
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.TreeSet;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.store.RAMDirectory;
+
+/**
+ * Tests segmented field data storage (files *.fdt, *.fd1, *.fd2 etc.)
+ * 
+ * @author Christian Kohlschuetter
+ */
+public class TestFieldDataStore extends TestCase {
+    private Analyzer a = new WhitespaceAnalyzer();
+
+    /*
+     * @see TestCase#setUp()
+     */
+    protected void setUp() throws Exception {
+        super.setUp();
+    }
+
+    /*
+     * @see TestCase#tearDown()
+     */
+    protected void tearDown() throws Exception {
+        super.tearDown();
+    }
+
+    public void test1_regular() throws IOException {
+        internal1(false);
+    }
+
+    public void test1_compound() throws IOException {
+        internal1(true);
+    }
+
+    private void internal1(boolean useCompound) throws IOException {
+        RAMDirectory dir = new RAMDirectory();
+
+        IndexWriter iw = new IndexWriter(dir, a, true);
+        iw.setUseCompoundFile(useCompound);
+
+        Document doc;
+
+        /**
+         * Create three test documents
+         * 
+         * Document 1 and 3 are regular (docStore is always == 0)
+         * 
+         * Document 2 contains nine fields: two in docStore 0, three in
+         * docStore 1 and four in docStore 2
+         */
+
+        doc = new Document();
+        doc.add(Field.Keyword("regular", "doc1"));
+        iw.addDocument(doc);
+
+        doc = new Document();
+        doc.add(Field.Keyword("FIELD_0_STORE_0", "0/0"));
+        doc.add(Field.Keyword("FIELD_1_STORE_0", "1/0"));
+        doc.add(Field.Keyword("FIELD_0_STORE_1", "0/1").setDataStore(1));
+        doc.add(Field.Keyword("FIELD_1_STORE_1", "1/1").setDataStore(1));
+        doc.add(Field.Keyword("FIELD_2_STORE_1", "2/1").setDataStore(1));
+        doc.add(Field.Keyword("FIELD_0_STORE_2", "0/2").setDataStore(2));
+        doc.add(Field.Keyword("FIELD_1_STORE_2", "1/2").setDataStore(2));
+        doc.add(Field.Keyword("FIELD_2_STORE_2", "2/2").setDataStore(2));
+        doc.add(Field.Keyword("FIELD_3_STORE_2", "3/2").setDataStore(2));
+        iw.addDocument(doc);
+
+        /**
+         * This document cannot be added since the dataStoreID is
+         * non-contiguous. We will get an IOException when calling <code>addDocument(doc)</code>
+         */
+        doc = new Document();
+        doc.add(Field.Keyword("regular", "doc"));
+        doc.add(Field.Keyword("BADFIELD", "99").setDataStore(99));
+
+        boolean ok = false;
+        try {
+            iw.addDocument(doc);
+        } catch (IOException e) {
+            ok = true;
+        }
+        assertTrue("Did not receive expected IOException", ok);
+
+        /**
+         * A regular document again
+         */
+        doc = new Document();
+        doc.add(Field.Keyword("regular", "doc2"));
+        iw.addDocument(doc);
+
+        iw.close();
+
+        /**
+         * Now check the index
+         */
+        IndexReader ir = IndexReader.open(dir);
+        Enumeration e;
+        Field f;
+        assertEquals(3, ir.numDocs());
+
+        TreeSet tsExpected = new TreeSet(Arrays.asList(new String[] {
+            "", // is this correct?
+            "FIELD_0_STORE_0", "FIELD_1_STORE_0", "FIELD_0_STORE_1",
+            "FIELD_1_STORE_1", "FIELD_2_STORE_1", "FIELD_0_STORE_2",
+            "FIELD_1_STORE_2", "FIELD_2_STORE_2", "FIELD_3_STORE_2", "regular"
+        }));
+
+        TreeSet tsReal = new TreeSet(ir.getFieldNames());
+        assertEquals(tsExpected, tsReal);
+
+        /**
+         * Checks a regular document with all field data in the standard .fdt
+         * file (field store 0)
+         */
+        doc = ir.document(0);
+        e = doc.fields();
+        f = (Field) e.nextElement();
+        assertEquals("regular", f.name());
+        assertEquals("doc1", f.stringValue());
+        assertEquals(0, f.getDataStore());
+        assertFalse(e.hasMoreElements());
+
+        /**
+         * Checks ir.document(1,0) and ir.document(1,1) and ir.document(1,2)
+         */
+        for(int sm = 0; sm <= 2; sm++) {
+            doc = ir.document(1, sm);
+            e = doc.fields();
+            for(int sn = 0; sn <= sm; sn++) {
+                for(int fn = 0; fn < 2 + sn * 1; fn++) {
+                    f = (Field) e.nextElement();
+                    assertEquals("FIELD_" + fn + "_STORE_" + sn, f.name());
+                    assertEquals(fn + "/" + sn, f.stringValue());
+                    assertEquals(sn, f.getDataStore());
+                }
+            }
+
+            /**
+             * The following assert will fail with sm==0 or sm==1 if a
+             * docStore-unaware IndexReader is used as in that case, all field
+             * stores are read (see {@link IndexReader.document(int, int)})
+             * 
+             * Solution: Overload {@link IndexReader.document(int, int)}in the
+             * specific subclass.
+             */
+            assertFalse(e.hasMoreElements());
+        }
+
+        /**
+         * Checks ir.document(1), which is equivalent to ir.document(1,2) in
+         * this test scenario
+         */
+        doc = ir.document(1);
+        e = doc.fields();
+        for(int sn = 0; sn <= 2; sn++) {
+            for(int fn = 0; fn < 2 + sn * 1; fn++) {
+                f = (Field) e.nextElement();
+                assertEquals("FIELD_" + fn + "_STORE_" + sn, f.name());
+                assertEquals(fn + "/" + sn, f.stringValue());
+                assertEquals(sn, f.getDataStore());
+            }
+        }
+        assertFalse(e.hasMoreElements());
+
+        /**
+         * Checks a regular document with all field data in the standard .fdt
+         * file (field store 0)
+         */
+        doc = ir.document(2);
+        e = doc.fields();
+        f = (Field) e.nextElement();
+        assertEquals("regular", f.name());
+        assertEquals("doc2", f.stringValue());
+        assertEquals(0, f.getDataStore());
+        assertFalse(e.hasMoreElements());
+
+        /**
+         * Check directory contents
+         * 
+         * This sub-test will fail when new file types are introduced.
+         */
+        String[] names = dir.list();
+
+        TreeSet files = getSuffixes(names);
+
+        TreeSet expectedFiles;
+        if(useCompound) {
+            expectedFiles = new TreeSet(Arrays.asList(new String[] {
+                "deletable", "segments", "cfs"
+            }));
+
+            assertEquals(expectedFiles, files);
+            
+            String compoundFile = null;
+            for(int i=0;i<names.length;i++) {
+                if(names[i].endsWith(".cfs")) {
+                    compoundFile = names[i];
+                    break;
+                }
+            }
+            
+            CompoundFileReader cfr = new CompoundFileReader(dir, compoundFile);
+            names = cfr.list();
+            files = getSuffixes(names);
+            
+            /** These two files are not stored in the compound file */
+            files.add("deletable");
+            files.add("segments");
+            
+            checkExpectedSuffixes(files);
+        } else {
+            checkExpectedSuffixes(files);
+        }
+
+        ir.close();
+    }
+    
+    private TreeSet getSuffixes(String[] filenames) {
+        TreeSet files = new TreeSet();
+        for(int i = 0; i < filenames.length; i++) {
+            String name = filenames[i];
+            int dot = name.indexOf('.');
+            if(dot != -1) {
+                name = name.substring(dot + 1);
+            }
+            files.add(name);
+        }
+        return files;
+    }
+    
+    private void checkExpectedSuffixes(TreeSet files) {
+        TreeSet expectedFiles = new TreeSet(Arrays.asList(new String[] {
+            "deletable", "segments", "fdx", "fnm", "frq", "prx", "tii",
+            "tis"
+        }));
+
+        for(int i = 1; i <= 10; i++) {
+            expectedFiles.add("f" + i);
+        }
+        expectedFiles.add("fdt");
+        expectedFiles.add("fd1");
+        expectedFiles.add("fd2");
+        
+        assertEquals(expectedFiles, files);
+    }
+}