Index: lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java	(revision 1428681)
+++ lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java	(working copy)
@@ -82,7 +82,7 @@
     final SegmentInfo si = new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, -1, false, codec, null, null);
 
     SegmentMerger merger = new SegmentMerger(si, InfoStream.getDefault(), mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL,
-                                             MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()));
+                                             MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()), null);
     merger.add(reader1);
     merger.add(reader2);
     MergeState mergeState = merger.merge();
Index: lucene/core/src/test/org/apache/lucene/index/TestDoc.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestDoc.java	(revision 1428681)
+++ lucene/core/src/test/org/apache/lucene/index/TestDoc.java	(working copy)
@@ -219,7 +219,7 @@
       final SegmentInfo si = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged, -1, false, codec, null, null);
 
       SegmentMerger merger = new SegmentMerger(si, InfoStream.getDefault(), trackingDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL,
-                                               MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), context);
+                                               MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), context, null);
 
       merger.add(r1);
       merger.add(r2);
Index: lucene/core/src/test/org/apache/lucene/index/TestExternalSegmentData.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestExternalSegmentData.java	(revision 0)
+++ lucene/core/src/test/org/apache/lucene/index/TestExternalSegmentData.java	(working copy)
@@ -0,0 +1,177 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+public class TestExternalSegmentData extends LuceneTestCase {
+
+  private static final class StoredFieldCountPerDocument extends ExternalSegmentData {
+
+    @Override
+    public SegmentWriter newSegment(String segmentName) {
+      return new SegmentWriter() {
+        private final List<Integer> fieldCounts = new ArrayList<Integer>();
+
+        @Override
+        public void addDocument(int docID, IndexDocument doc) {
+          // Skip any missing (deleted due to exc during
+          // indexing) docs:
+          while(fieldCounts.size() < docID) {
+            fieldCounts.add(0);
+          }
+          int count = 0;
+          for(StorableField f : doc.storableFields()) {
+            if (f.fieldType().stored()) {
+              // RIW randomly adds not-stored DocValues
+              // fields, so we have to not count those
+              count++;
+            }
+          }
+          if (VERBOSE) {
+            System.out.println("FC: add docID=" + docID + " count=" + count);
+          }
+          fieldCounts.add(count);
+        }
+
+        @Override
+        public void flush(SegmentWriteState state) throws IOException {
+          if (VERBOSE) {
+            System.out.println("FC: flush seg=" + state.segmentInfo.name + " dir=" + state.directory);
+          }
+          while(fieldCounts.size() < state.segmentInfo.getDocCount()) {
+            fieldCounts.add(0);
+          }
+          String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, "", "fc");
+          IndexOutput o = state.directory.createOutput(fileName, IOContext.DEFAULT);
+          for(int size : fieldCounts) {
+            if (VERBOSE) {
+              System.out.println("  write " + size);
+            }
+            o.writeVInt(size);
+          }
+          o.close();
+        }
+      };
+    }
+
+    @Override
+    public void merge(SegmentWriteState state, MergeState mergeState) throws IOException {
+      String mergedFileName = IndexFileNames.segmentFileName(mergeState.segmentInfo.name, "", "fc");
+      if (VERBOSE) {
+        System.out.println("FC: merge seg=" + mergeState.segmentInfo.name);
+      }
+      IndexOutput o = state.directory.createOutput(mergedFileName, IOContext.DEFAULT);
+      
+      try {
+        int readerIDX = 0;
+        for(AtomicReader r : mergeState.readers) {
+          // NOTE: will not work if you eg addIndices(SlowCompositeReader...)
+          String fileName = IndexFileNames.segmentFileName(((SegmentReader) r).getSegmentName(), "", "fc");
+          IndexInput i = state.directory.openInput(fileName, IOContext.DEFAULT);
+          try {
+            Bits liveDocs = r.getLiveDocs();
+            for(int docID=0;docID<r.maxDoc();docID++) {
+              int count = i.readVInt();
+              if (liveDocs == null || liveDocs.get(docID)) {
+                o.writeVInt(count);
+              }
+            }
+            readerIDX++;
+          } finally {
+            i.close();
+          }
+        }
+      } finally {
+        o.close();
+      }
+    }
+  }
+
+  public void test() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    iwc.setExternalSegmentData(new StoredFieldCountPerDocument());
+    TieredMergePolicy tmp = new TieredMergePolicy();
+    // We could allow true if we fixed our "reader" to open
+    // CFS when necessary:
+    tmp.setUseCompoundFile(false);
+    iwc.setMergePolicy(tmp);
+
+    // nocommit add threads here:
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    int numDocs = atLeast(1000);
+    for(int i=0;i<numDocs;i++) {
+      int numFields = _TestUtil.nextInt(random(), 1, 20);
+      Document doc = new Document();
+      for(int j=0;j<numFields;j++) {
+        doc.add(new StringField(""+j, "foo", Field.Store.YES));
+      }
+      if (VERBOSE) {
+        System.out.println("TEST: add doc numFields=" + numFields);
+      }
+      w.addDocument(doc);
+    }
+
+    // nocommit not great that we need to do this hackery
+    // ... it's only because new SegmentReader doens't open
+    // our custom files ... and so a merge that finishes may
+    // delete them after w.getReader() and before we open
+    // them ourselves ... need some basic read-time API:
+    w.commit();
+    w.w.waitForMerges();
+
+    IndexReader r = w.getReader();
+    w.close();
+
+    for(AtomicReaderContext ctx : r.leaves()) {
+      SegmentReader subR = (SegmentReader) ctx.reader();
+      String fileName = IndexFileNames.segmentFileName(((SegmentReader) subR).getSegmentName(), "", "fc");
+      IndexInput i = dir.openInput(fileName, IOContext.DEFAULT);
+      if (VERBOSE) {
+        System.out.println("FC: open " + fileName);
+      }
+      for(int docID=0;docID<subR.maxDoc();docID++) {
+        if (VERBOSE) {
+          System.out.println("FC: test doc=" + docID);
+        }
+        StoredDocument doc = subR.document(docID);
+        assertEquals(doc.getFields().size(), i.readVInt());
+      }
+      i.close();
+    }
+
+    r.close();
+    dir.close();
+  }
+  
+}

Property changes on: lucene/core/src/test/org/apache/lucene/index/TestExternalSegmentData.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java	(revision 1428681)
+++ lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java	(working copy)
@@ -50,6 +50,9 @@
    *  points are deleted. */
   protected volatile IndexDeletionPolicy delPolicy;
 
+  /** @lucene.experimental */
+  protected volatile ExternalSegmentData esd;
+
   /** {@link IndexCommit} that {@link IndexWriter} is
    *  opened on. */
   protected volatile IndexCommit commit;
@@ -151,6 +154,7 @@
     readerPooling = config.getReaderPooling();
     flushPolicy = config.getFlushPolicy();
     perThreadHardLimitMB = config.getRAMPerThreadHardLimitMB();
+    esd = config.getExternalSegmentData();
   }
 
   /** Returns the default analyzer to use for indexing documents. */
@@ -539,6 +543,11 @@
     return infoStream;
   }
   
+  /** @lucene.experimental */
+  public ExternalSegmentData getExternalSegmentData() {
+    return esd;
+  }
+
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder();
Index: lucene/core/src/java/org/apache/lucene/index/ExternalSegmentData.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/ExternalSegmentData.java	(revision 0)
+++ lucene/core/src/java/org/apache/lucene/index/ExternalSegmentData.java	(working copy)
@@ -0,0 +1,41 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+/** Expert: use this to write custom, private per-segment
+ *  data during indexing.  One can do this today by making
+ *  a custom indexing chain, but this solution is much
+ *  simpler but less flexible.
+ *
+ * @lucene.experimental */
+
+public abstract class ExternalSegmentData {
+
+  public static abstract class SegmentWriter {
+    /** Single threaded ... NOTE: because of possible exceptions, some docIDs
+     *  may be 'skipped' (and deleted) from this API: */
+    public abstract void addDocument(int docID, IndexDocument doc) throws IOException;
+    public abstract void flush(SegmentWriteState state) throws IOException;
+  }
+
+  public abstract SegmentWriter newSegment(String segmentName);
+
+  public abstract void merge(SegmentWriteState state, MergeState mergeState) throws IOException;
+}

Property changes on: lucene/core/src/java/org/apache/lucene/index/ExternalSegmentData.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java	(revision 1428681)
+++ lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java	(working copy)
@@ -416,6 +416,12 @@
     return this;
   }
 
+  /** @lucene.experimental */
+  public IndexWriterConfig setExternalSegmentData(ExternalSegmentData esd) {
+    this.esd = esd;
+    return this;
+  }
+
   @Override
   public int getRAMPerThreadHardLimitMB() {
     return perThreadHardLimitMB;
Index: lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java	(revision 1428681)
+++ lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java	(working copy)
@@ -188,7 +188,9 @@
   private final NumberFormat nf = NumberFormat.getInstance(Locale.ROOT);
   final Allocator byteBlockAllocator;
   final IntBlockPool.Allocator intBlockAllocator;
+  final ExternalSegmentData esd;
 
+  private ExternalSegmentData.SegmentWriter esdSegment;
   
   public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent,
       FieldInfos.Builder fieldInfos, IndexingChain indexingChain) {
@@ -201,6 +203,7 @@
     this.codec = parent.codec;
     this.docState = new DocState(this, infoStream);
     this.docState.similarity = parent.indexWriter.getConfig().getSimilarity();
+    this.esd = parent.indexWriter.getConfig().getExternalSegmentData();
     bytesUsed = Counter.newCounter();
     byteBlockAllocator = new DirectTrackingAllocator(bytesUsed);
     pendingDeletes = new BufferedDeletes();
@@ -248,6 +251,10 @@
     try {
       try {
         consumer.processDocument(fieldInfos);
+        // nocommit move into consumer?
+        if (esdSegment != null) {
+          esdSegment.addDocument(docState.docID, doc);
+        }
       } finally {
         docState.clear();
       }
@@ -283,6 +290,11 @@
     if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
       infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);  
     }
+    if (esd != null) {
+      esdSegment = esd.newSegment(segmentInfo.name);
+    } else {
+      esdSegment = null;
+    }
   }
   
   public int updateDocuments(Iterable<? extends IndexDocument> docs, Analyzer analyzer, Term delTerm) throws IOException {
@@ -307,6 +319,10 @@
         boolean success = false;
         try {
           consumer.processDocument(fieldInfos);
+          // nocommit move into consumer?
+          if (esdSegment != null) {
+            esdSegment.addDocument(docState.docID, doc);
+          }
           success = true;
         } finally {
           if (!success) {
@@ -486,6 +502,10 @@
 
     try {
       consumer.flush(flushState);
+      if (esdSegment != null) {
+        esdSegment.flush(flushState);
+      }
+
       pendingDeletes.terms.clear();
       segmentInfo.setFiles(new HashSet<String>(directory.getCreatedFiles()));
 
Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java	(revision 1428681)
+++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java	(working copy)
@@ -2414,7 +2414,8 @@
                                          false, codec, null, null);
 
       SegmentMerger merger = new SegmentMerger(info, infoStream, trackingDir, config.getTermIndexInterval(),
-                                               MergeState.CheckAbort.NONE, globalFieldNumberMap, context);
+                                               MergeState.CheckAbort.NONE, globalFieldNumberMap, context,
+                                               config.getExternalSegmentData());
 
       for (IndexReader reader : readers) {    // add new indexes
         merger.add(reader);
@@ -3565,7 +3566,7 @@
     final TrackingDirectoryWrapper dirWrapper = new TrackingDirectoryWrapper(directory);
 
     SegmentMerger merger = new SegmentMerger(merge.info.info, infoStream, dirWrapper, config.getTermIndexInterval(), checkAbort,
-                                             globalFieldNumberMap, context);
+                                             globalFieldNumberMap, context, config.getExternalSegmentData());
 
     if (infoStream.isEnabled("IW")) {
       infoStream.message("IW", "merging " + segString(merge.segments));
Index: lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java	(revision 1428681)
+++ lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java	(working copy)
@@ -54,9 +54,12 @@
   private final MergeState mergeState = new MergeState();
   private final FieldInfos.Builder fieldInfosBuilder;
 
+  private final ExternalSegmentData esd;
+
   // note, just like in codec apis Directory 'dir' is NOT the same as segmentInfo.dir!!
   SegmentMerger(SegmentInfo segmentInfo, InfoStream infoStream, Directory dir, int termIndexInterval,
-                MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context) {
+                MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context,
+                ExternalSegmentData esd) {
     mergeState.segmentInfo = segmentInfo;
     mergeState.infoStream = infoStream;
     mergeState.readers = new ArrayList<AtomicReader>();
@@ -66,6 +69,7 @@
     this.codec = segmentInfo.getCodec();
     this.context = context;
     this.fieldInfosBuilder = new FieldInfos.Builder(fieldNumbers);
+    this.esd = esd;
   }
 
   /**
@@ -120,6 +124,10 @@
     FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
     fieldInfosWriter.write(directory, mergeState.segmentInfo.name, mergeState.fieldInfos, context);
 
+    if (esd != null) {
+      esd.merge(segmentWriteState, mergeState);
+    }
+
     return mergeState;
   }