Index: lucene/core/src/test/org/apache/lucene/search/TestLiveFieldValues.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/TestLiveFieldValues.java	(revision 0)
+++ lucene/core/src/test/org/apache/lucene/search/TestLiveFieldValues.java	(working copy)
@@ -0,0 +1,165 @@
+package org.apache.lucene.search;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.NRTManager.TrackingIndexWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+public class TestLiveFieldValues extends LuceneTestCase {
+  public void test() throws Exception {
+
+    Directory dir = newFSDirectory(_TestUtil.getTempDir("livefieldupdates"));
+    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+
+    final IndexWriter _w = new IndexWriter(dir, iwc);
+    final TrackingIndexWriter w = new TrackingIndexWriter(_w);
+
+    final NRTManager mgr = new NRTManager(w, new SearcherFactory() {
+        @Override
+        public IndexSearcher newSearcher(IndexReader r) {
+          return new IndexSearcher(r);
+        }
+      });
+
+    final Integer missing = -1;
+
+    final LiveFieldValues<Integer> rt = new LiveFieldValues<Integer>("id", "field", mgr, missing) {
+        @Override
+        protected Integer lookupFromSearcher(IndexSearcher s, String id) throws IOException {
+          TermQuery tq = new TermQuery(new Term("id", id));
+          TopDocs hits = s.search(tq, 1);
+          assertTrue(hits.totalHits <= 1);
+          if (hits.totalHits == 0) {
+            return null;
+          } else {
+            StoredDocument doc = s.doc(hits.scoreDocs[0].doc);
+            return (Integer) doc.getField("field").numericValue();
+          }
+        }
+    };
+
+    int numThreads = _TestUtil.nextInt(random(), 2, 5);
+    if (VERBOSE) {
+      System.out.println(numThreads + " threads");
+    }
+
+    final CountDownLatch startingGun = new CountDownLatch(1);
+    List<Thread> threads = new ArrayList<Thread>();
+
+    final int iters = atLeast(1000);
+
+    for(int t=0;t<numThreads;t++) {
+      final int threadID = t;
+      Thread thread = new Thread() {
+
+          @Override
+          public void run() {
+            try {
+              Map<String,Integer> values = new HashMap<String,Integer>();
+              List<String> allIDs = Collections.synchronizedList(new ArrayList<String>());
+
+              startingGun.await();
+              for(int iter=0; iter<iters;iter++) {
+                // Add/update a document
+                Document doc = new Document();
+                // Threads must not update the same id at the
+                // same time:
+                String id = String.format("%d_%04x", threadID, random().nextInt(65536));
+                Integer field = random().nextInt(Integer.MAX_VALUE);
+                doc.add(new StringField("id", id, Field.Store.YES));
+                doc.add(new IntField("field", field.intValue(), Field.Store.YES));
+                long gen = w.updateDocument(new Term("id", id), doc);
+                rt.add(gen, id, field);
+                if (values.put(id, field) == null) {
+                  allIDs.add(id);
+                }
+
+                if (random().nextInt(4) == 1) {
+                  String randomID = allIDs.get(random().nextInt(allIDs.size()));
+                  gen = w.deleteDocuments(new Term("id", randomID));
+                  rt.delete(gen, randomID);
+                  values.put(randomID, missing);
+                }
+
+                if (random().nextInt(100) == 17) {
+                  mgr.maybeRefresh();
+                  if (VERBOSE) {
+                    IndexSearcher s = mgr.acquire();
+                    try {
+                      System.out.println("TEST: reopen " + s);
+                    } finally {
+                      mgr.release(s);
+                    }
+                    System.out.println("TEST: " + values.size() + " values");
+                  }
+                }
+
+                // nocommit sometimes lookup non-existent id...
+
+                String randomID = allIDs.get(random().nextInt(allIDs.size()));
+                Integer expected = values.get(randomID);
+                if (expected == missing) {
+                  expected = null;
+                }
+                assertEquals("id=" + randomID, expected, rt.get(randomID));
+              }
+            } catch (Throwable t) {
+              throw new RuntimeException(t);
+            }
+          }
+        };
+      threads.add(thread);
+      thread.start();
+    }
+
+    startingGun.countDown();
+
+    for(Thread thread : threads) {
+      thread.join();
+    }
+    mgr.maybeRefresh();
+    assertEquals(0, rt.size());
+
+    rt.close();
+    mgr.close();
+    _w.close();
+    dir.close();
+  }
+}

Property changes on: lucene/core/src/test/org/apache/lucene/search/TestLiveFieldValues.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/core/src/java/org/apache/lucene/search/LiveFieldValues.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/search/LiveFieldValues.java	(revision 0)
+++ lucene/core/src/java/org/apache/lucene/search/LiveFieldValues.java	(working copy)
@@ -0,0 +1,147 @@
+package org.apache.lucene.search;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.IndexDocument;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.Counter;
+
+/** Tracks live field values across NRT reader reopens.
+ *  This holds a map for all updated ids since
+ *  the last reader reopen.  Once the NRT reader is reopened,
+ *  it prunes the map.  This means you must reopen your NRT
+ *  reader periodically otherwise the RAM consumption of
+ *  this class will grow unbounded!
+ *
+ *  <p>NOTE: you must ensure the same id is never updated at
+ *  the same time by two threads, because in this case you
+ *  cannot in general know which thread "won". */
+
+// nocommit parameterize id field type too?
+public abstract class LiveFieldValues<T> implements ReferenceManager.RefreshListener, Closeable {
+
+  private final Map<String,T> idToFieldValue = new ConcurrentHashMap<String,T>();
+  private final Map<String,Long> idToGen = new ConcurrentHashMap<String,Long>();
+  private final NRTManager mgr;
+  private final String fieldName;
+  private final String idFieldName;
+  private final T missingValue;
+
+  public LiveFieldValues(String idFieldName, String fieldName, NRTManager mgr, T missingValue) {
+    this.fieldName = fieldName;
+    this.idFieldName = idFieldName;
+    this.mgr = mgr;
+    this.missingValue = missingValue;
+    mgr.addListener(this);
+  }
+
+  @Override
+  public void close() {
+    mgr.removeListener(this);
+  }
+
+  @Override
+  public void afterRefresh() throws IOException {
+    prune();
+  }
+
+  private void prune() throws IOException {
+    final long nowSearching = mgr.getCurrentSearchingGen();
+    //System.out.println("RT: prune <= gen=" + nowSearching);
+
+    // Prune all entries <= current gen:
+    int count = 0;
+    Iterator<Map.Entry<String,Long>> it = idToGen.entrySet().iterator();
+    while(it.hasNext()) {
+      Map.Entry<String,Long> ent = it.next();
+      if (ent.getValue() <= nowSearching) {
+        it.remove();
+        idToFieldValue.remove(ent.getKey());
+        count++;
+      }
+    }
+    //System.out.println("  pruned " + count + " ids; now " + idToFieldValue.size() + " RAM ids");
+  }
+
+  /** Call this after you've successfully added a document
+   *  to the index, to record what value you just set the
+   *  field to. */
+  public void add(long gen, String id, T value) {
+    idToFieldValue.put(id, value);
+    idToGen.put(id, gen);
+  }
+
+  /** Call this after you've successfully deleted a document
+   *  from the index. */
+  public void delete(long gen, String id) {
+    idToFieldValue.put(id, missingValue);
+    idToGen.put(id, gen);
+  }
+
+  /** Returns the [approximate] number of id/value pairs
+   *  buffered in RAM. */
+  public int size() {
+    return idToFieldValue.size();
+  }
+
+  /** Returns the current value for this id, or null if the
+   *  id isn't in the index or was deleted. */
+  public T get(String id) throws IOException {
+    // First try to get the "live" value:
+    T value = idToFieldValue.get(id);
+    if (value == missingValue) {
+      // Deleted but the deletion is not yet reflected in
+      // the reader:
+      return null;
+    } else if (value != null) {
+      return value;
+    } else {
+      // It either does not exist in the index, or, it was
+      // already flushed & NRT reader was opened on the
+      // segment, so fallback to current searcher:
+      IndexSearcher s = mgr.acquire();
+      try {
+        return lookup(s, id);
+      } finally {
+        mgr.release(s);
+      }
+    }
+  }
+
+  /** This is called when the id/value was already flushed & opened
+   *  in an NRT IndexSearcher.  You must implement this to
+   *  go look up the value (eg, via doc values, field cache,
+   *  stored fields, etc.). */
+  protected abstract T lookupFromSearcher(IndexSearcher s, String id) throws IOException;
+}
+

Property changes on: lucene/core/src/java/org/apache/lucene/search/LiveFieldValues.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java	(revision 1434876)
+++ lucene/core/src/java/org/apache/lucene/search/ReferenceManager.java	(working copy)
@@ -254,7 +254,7 @@
     decRef(reference);
   }
 
-  private void notifyRefreshListeners() {
+  private void notifyRefreshListeners() throws IOException {
     for (RefreshListener refreshListener : refreshListeners) {
       refreshListener.afterRefresh();
     }
@@ -287,6 +287,6 @@
     /**
      * Called after a successful refresh and a new reference has been installed. When this is called {@link #acquire()} is guaranteed to return a new instance.
      */
-    void afterRefresh();
+    void afterRefresh() throws IOException;
   }
 }