Index: lucene/src/test/org/apache/lucene/search/TestShardSearching.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/TestShardSearching.java	(revision 0)
+++ lucene/src/test/org/apache/lucene/search/TestShardSearching.java	(working copy)
@@ -0,0 +1,183 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util._TestUtil;
+
+// TODO
+//   - test searchAfter too, w/ sharding!
+
+public class TestShardSearching extends ShardSearchingTestBase {
+
+  public void testSimple() throws Exception {
+    // TODO: randomize:
+    start(_TestUtil.getTempDir("TestShardSearching").toString(),
+          _TestUtil.nextInt(random, 1, 10),     // numShards
+          10.0*RANDOM_MULTIPLIER,
+          _TestUtil.nextInt(random, 1, 4)       // maxSearcherAgeSeconds
+          );
+    final List<long[]> lastVersions = new ArrayList<long[]>();
+    List<BytesRef> terms = null;
+    while (System.nanoTime() < endTimeNanos) {
+      final SearcherAndVersion[] searchers;
+      // TODO: make this smart, not just lastVersion, but eg
+      // save some really old versions and rerun, so that we
+      // sometimes hit timeout
+
+      final boolean doFollowon = lastVersions.size() > 0 && random.nextInt(7) == 1;
+
+      if (doFollowon) {
+        final long[] lastVersion = lastVersions.get(random.nextInt(lastVersions.size()));
+        // Do followon query; could be "old"!
+        try {
+          searchers = acquire(lastVersion);
+        } catch (SearcherExpiredException see) {
+          // Expected, sometimes; in a "real" app we would
+          // either forward this error to the user ("too
+          // much time has passed; please re-run your
+          // search") or sneakily just switch to newest
+          // searcher w/o telling them...
+          lastVersions.remove(lastVersion);
+          continue;
+        }
+      } else {
+        // Do fresh query:
+        searchers = acquire();
+      }
+
+      final long[] shardVersions = justVersion(searchers);
+
+      try {
+
+        // Make single (MultiReader) from all searchers:
+        int docCount = 0;
+        final IndexReader[] subs = new IndexReader[searchers.length];
+        for(int subID=0;subID<subs.length;subID++) {
+          subs[subID] = searchers[subID].searcher.getIndexReader();
+          docCount += subs[subID].maxDoc();
+        }
+        final IndexReader singleReader = new MultiReader(subs);
+        final IndexSearcher singleSearcher = new IndexSearcher(singleReader);
+
+        if (terms == null && docCount > 10) {
+          // TODO: try to "focus" on high freq terms sometimes too
+          // TODO: maybe also periodically reset the terms...?
+          final TermsEnum termsEnum = MultiFields.getTerms(singleReader, "body").iterator(null);
+          terms = new ArrayList<BytesRef>();
+          while(termsEnum.next() != null) {
+            terms.add(BytesRef.deepCopyOf(termsEnum.term()));
+          }
+          System.out.println(terms.size() + " terms");
+          if (terms.size() == 0) {
+            terms = null;
+          }
+        }
+
+        if (terms != null) {
+          TermQuery query = new TermQuery(new Term("body", terms.get(random.nextInt(terms.size()))));
+
+          try {
+            assertSame(singleSearcher, searchers, shardVersions, query);
+          } catch (SearcherExpiredException see) {
+            // Expected, sometimes; in a "real" app we would
+            // either forward this error to the user ("too
+            // much time has passed; please re-run your
+            // search") or sneakily just switch to newest
+            // searcher w/o telling them...
+            assert lastVersions.indexOf(shardVersions) != -1;
+            lastVersions.remove(shardVersions);
+          }
+        }
+      } finally {
+        release(searchers);
+      }
+
+      if (!doFollowon && random.nextInt(50) == 17) {
+        lastVersions.add(shardVersions);
+        if (lastVersions.size() > 200) {
+          Collections.shuffle(lastVersions, random);
+          lastVersions.subList(100, lastVersions.size()).clear();
+        }
+      }
+    }
+
+    finish();
+  }
+
+  private void assertSame(IndexSearcher singleSearcher, SearcherAndVersion[] shardSearchers, long[] shardVersions, Query q) throws IOException {
+    final int numHits = _TestUtil.nextInt(random, 1, 100);
+
+    // Single searcher:
+    TopDocs hits = singleSearcher.search(q, numHits);
+
+    // Sharded searcher:
+    final TopDocs[] shardHits = new TopDocs[shardSearchers.length];
+    for(int shardID=0;shardID<shardSearchers.length;shardID++) {
+      final IndexSearcher shardSearcher = new MockSlowShardIndexSearcher(shardVersions, shardSearchers[shardID].searcher.getIndexReader(), shardID);
+      shardHits[shardID] = shardSearcher.search(q, numHits);
+    }
+
+    // nocommit throw in random sort / filter in:
+
+    // Merge:
+    final TopDocs mergedHits = TopDocs.merge(null, numHits, shardHits);
+
+    if (VERBOSE) {
+      System.out.println("TEST: query=" + q);
+      /*
+      for(int shardID=0;shardID<shardSearchers.length;shardID++) {
+        System.out.println("  shard=" + shardID + " maxDoc=" + shardSearchers[shardID].searcher.getIndexReader().maxDoc());
+      }
+      */
+      System.out.println("  single searcher: " + hits.totalHits + " totalHits maxScore=" + hits.getMaxScore());
+      for(int i=0;i<hits.scoreDocs.length;i++) {
+        final ScoreDoc sd = hits.scoreDocs[i];
+        System.out.println("    doc=" + sd.doc + " score=" + sd.score);
+      }
+      System.out.println("  shard searcher: " + mergedHits.totalHits + " totalHits maxScore=" + mergedHits.getMaxScore());
+      for(int i=0;i<mergedHits.scoreDocs.length;i++) {
+        final ScoreDoc sd = mergedHits.scoreDocs[i];
+        System.out.println("    doc=" + sd.doc + " score=" + sd.score + " shard=" + sd.shardIndex);
+      }
+    }
+
+    // Must rebase so assertEquals passes:
+    int[] base = new int[shardSearchers.length];
+    int docCount = 0;
+    for(int shardID=0;shardID<shardSearchers.length;shardID++) {
+      base[shardID] = docCount;
+      docCount += shardSearchers[shardID].searcher.getIndexReader().maxDoc();
+    }
+    for(int hitID=0;hitID<mergedHits.scoreDocs.length;hitID++) {
+      final ScoreDoc sd = mergedHits.scoreDocs[hitID];
+      sd.doc += base[sd.shardIndex];
+    }
+    _TestUtil.assertEquals(hits, mergedHits);
+  }
+}

Property changes on: lucene/src/test/org/apache/lucene/search/TestShardSearching.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
Index: lucene/src/java/org/apache/lucene/search/TopDocs.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/TopDocs.java	(revision 1213036)
+++ lucene/src/java/org/apache/lucene/search/TopDocs.java	(working copy)
@@ -225,6 +225,10 @@
       }
     }
 
+    if (maxScore == Float.MIN_VALUE) {
+      maxScore = Float.NaN;
+    }
+
     final ScoreDoc[] hits = new ScoreDoc[Math.min(topN, availHitCount)];
 
     int hitUpto = 0;
Index: lucene/src/test-framework/java/org/apache/lucene/search/ShardSearchingTestBase.java
===================================================================
--- lucene/src/test-framework/java/org/apache/lucene/search/ShardSearchingTestBase.java	(revision 0)
+++ lucene/src/test-framework/java/org/apache/lucene/search/ShardSearchingTestBase.java	(working copy)
@@ -0,0 +1,385 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.SearcherLifetimeManager;
+import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LineFileDocs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TermContext;
+import org.apache.lucene.util._TestUtil;
+
+// TODO
+//   - doc blocks?  so we can test joins/grouping...
+
+public abstract class ShardSearchingTestBase extends LuceneTestCase {
+
+  // nocommit make official; mabye SLM.acquire should throw
+  // this...?
+  // nocommit checked exception...?
+  public static class SearcherExpiredException extends RuntimeException {
+    public SearcherExpiredException(String message) {
+      super(message);
+    }
+  }
+
+  protected final class Index implements Closeable {
+    public final Directory dir;
+    public final RandomIndexWriter writer;
+    public final SearcherLifetimeManager searchers;
+    public final SearcherManager mgr;
+    public final int shardID;
+
+    public Index(String baseDir, int shardID) throws IOException {
+      this.shardID = shardID;
+      dir = newFSDirectory(new File(baseDir + "." + shardID));
+      // TODO: set warmer
+      writer = new RandomIndexWriter(random, dir);
+      mgr = new SearcherManager(writer.w, true, null);
+      searchers = new SearcherLifetimeManager();
+    }
+
+    public void reopen() throws IOException {
+      mgr.maybeReopen();
+      searchers.prune(new SearcherLifetimeManager.PruneByAge(maxSearcherAgeSeconds));
+    }
+
+    @Override
+    public void close() throws IOException {
+      searchers.close();
+      mgr.close();
+      writer.close();
+      dir.close();
+    }
+  }
+
+  private final class ChangeIndices extends Thread {
+    @Override
+    public void run() {
+      try {
+        final LineFileDocs docs = new LineFileDocs(random);
+        int numDocs = 0;
+        while (System.nanoTime() < endTimeNanos) {
+          final int what = random.nextInt(3);
+          final Index index = indices[random.nextInt(indices.length)];
+          if (numDocs == 0 || what == 0) {
+            index.writer.addDocument(docs.nextDoc());
+            numDocs++;
+          } else if (what == 1) {
+            index.writer.updateDocument(new Term("docid", ""+random.nextInt(numDocs)),
+                                        docs.nextDoc());
+            numDocs++;
+          } else {
+            index.writer.deleteDocuments(new Term("docid", ""+random.nextInt(numDocs)));
+          }
+          // TODO: doc blocks too
+
+          if (random.nextInt(17) == 12) {
+            index.writer.commit();
+          }
+
+          if (random.nextInt(17) == 12) {
+            indices[random.nextInt(indices.length)].reopen();
+          }
+        }
+      } catch (Throwable t) {
+        System.out.println("FAILED:");
+        t.printStackTrace(System.out);
+        throw new RuntimeException(t);
+      }
+    }
+  }
+
+  private Index[] indices;
+  int maxSearcherAgeSeconds;
+  long endTimeNanos;
+  private Thread changeIndicesThread;
+
+  protected void start(String baseDirName, int numShards, double runTimeSec, int maxSearcherAgeSeconds) throws IOException {
+
+    endTimeNanos = System.nanoTime() + (long) (runTimeSec*1000000000);
+    this.maxSearcherAgeSeconds = maxSearcherAgeSeconds;
+
+    indices = new Index[numShards];
+    for(int shardID=0;shardID<numShards;shardID++) {
+      indices[shardID] = new Index(baseDirName, shardID);
+    }
+
+    changeIndicesThread = new ChangeIndices();
+    changeIndicesThread.start();
+  }
+
+  protected void finish() throws InterruptedException, IOException {
+    changeIndicesThread.join();
+    for(Index index : indices) {
+      index.close();
+    }
+  }
+
+  protected static class SearcherAndVersion {
+    public final IndexSearcher searcher;
+    public final long version;
+
+    public SearcherAndVersion(IndexSearcher searcher, long version) {
+      this.searcher = searcher;
+      this.version = version;
+    }
+  }
+
+  /** Get fresh searchers. Make sure you release! */
+  protected SearcherAndVersion[] acquire() throws IOException {
+    final SearcherAndVersion[] searchers = new SearcherAndVersion[indices.length];
+    for(int shardID=0;shardID<searchers.length;shardID++) {
+      final IndexSearcher s = indices[shardID].mgr.acquire();
+      searchers[shardID] = new SearcherAndVersion(s, indices[shardID].searchers.record(s));
+    }
+    return searchers;
+  }
+
+  /** Silly but important utility!!  simulate round trip over
+   *  the network to user's browser and back... */
+  protected long[] justVersion(SearcherAndVersion[] searchers) {
+    final long[] versions = new long[searchers.length];
+    for(int shardID=0;shardID<searchers.length;shardID++) {
+      versions[shardID] = searchers[shardID].version;
+    }
+    return versions;
+  }
+
+  /** Get searchers matching prior version. Make sure you
+   *  release! */
+  protected SearcherAndVersion[] acquire(long[] versions) throws IOException {
+    assert versions.length == indices.length;
+    final SearcherAndVersion[] searchers = new SearcherAndVersion[indices.length];
+    for(int shardID=0;shardID<versions.length;shardID++) {
+      final long version = versions[shardID];
+      final IndexSearcher s = indices[shardID].searchers.acquire(version);
+      if (s == null) {
+        // TODO: make sure we test this...
+        shardID--;
+        while(shardID >= 0) {
+          indices[shardID].searchers.release(searchers[shardID].searcher);
+        }
+        throw new SearcherExpiredException("shardID=" + shardID + " version=" + version);
+      }
+      searchers[shardID] = new SearcherAndVersion(s, version);
+    }
+
+    return searchers;
+  }
+
+  protected void release(SearcherAndVersion[] searchers) throws IOException {
+    for(int shardID=0;shardID<searchers.length;shardID++) {
+      indices[shardID].mgr.release(searchers[shardID].searcher);
+    }
+  }
+
+  private static class FieldAndShardVersion {
+    private final long version;
+    private final int shardID;
+    private final String field;
+
+    public FieldAndShardVersion(int shardID, long version, String field) {
+      this.shardID = shardID;
+      this.version = version;
+      this.field = field;
+    }
+
+    @Override
+    public int hashCode() {
+      return (int) (version * shardID + field.hashCode());
+    }
+
+    @Override
+    public boolean equals(Object _other) {
+      if (!(_other instanceof FieldAndShardVersion)) {
+        return false;
+      }
+
+      final FieldAndShardVersion other = (FieldAndShardVersion) _other;
+
+      return field.equals(other.field) && version == other.version && shardID == other.shardID;
+    }
+  }
+
+  // TODO: nothing evicts from here!!!
+  private final Map<FieldAndShardVersion,CollectionStatistics> collectionStatsCache = new ConcurrentHashMap<FieldAndShardVersion,CollectionStatistics>();
+
+  // NOTE: in a "real" shard env this hits the wire to talk
+  // to the remote node.  We mock that up here by just
+  // querying our "local" shards.  Also, we would pass
+  // Set<String> fields, not one field at a time...:
+  CollectionStatistics getShardCollectionStats(int shardID, long version, String field) throws IOException {
+    final FieldAndShardVersion key = new FieldAndShardVersion(shardID, version, field);
+    CollectionStatistics stats = collectionStatsCache.get(key);
+    if (stats != null) {
+      // Cache hit
+      return stats;
+    }
+
+    // Cache miss
+    final IndexSearcher s = indices[shardID].searchers.acquire(version);
+    if (s == null) {
+      throw new SearcherExpiredException("shardID=" + shardID + " version=" + version);
+    }
+    try {
+      stats = s.collectionStatistics(field);
+    } finally {
+      indices[shardID].searchers.release(s);
+    }
+    collectionStatsCache.put(key, stats);
+    return stats;
+  }
+
+  private static class TermAndShardVersion {
+    private final long version;
+    private final int shardID;
+    private final Term term;
+
+    public TermAndShardVersion(int shardID, long version, Term term) {
+      this.shardID = shardID;
+      this.version = version;
+      this.term = term;
+    }
+
+    @Override
+    public int hashCode() {
+      return (int) (version * shardID + term.hashCode());
+    }
+
+    @Override
+    public boolean equals(Object _other) {
+      if (!(_other instanceof TermAndShardVersion)) {
+        return false;
+      }
+
+      final TermAndShardVersion other = (TermAndShardVersion) _other;
+
+      return term.equals(other.term) && version == other.version && shardID == other.shardID;
+    }
+  }
+
+  // TODO: nothing evicts from here!!!
+  private final Map<TermAndShardVersion,TermStatistics> termStatsCache = new ConcurrentHashMap<TermAndShardVersion,TermStatistics>();
+
+  // NOTE: in a "real" shard env this hits the wire to talk
+  // to the remote node.  We mock that up here by just
+  // querying our "local" shards:
+  TermStatistics getShardTermStats(int shardID, long version, Term term, TermContext termContext) throws IOException {
+    final TermAndShardVersion key = new TermAndShardVersion(shardID, version, term);
+    TermStatistics stats = termStatsCache.get(key);
+    if (stats != null) {
+      // Cache hit
+      return stats;
+    }
+
+    // Cache miss
+    final IndexSearcher s = indices[shardID].searchers.acquire(version);
+    if (s == null) {
+      throw new SearcherExpiredException("shardID=" + shardID + " version=" + version);
+    }
+    try {
+      // nocommit: is this right??? 
+      if (termContext == null) {
+        termContext = TermContext.build(s.getIndexReader().getTopReaderContext(), term, false);
+      }
+      stats = s.termStatistics(term, termContext);
+    } finally {
+      indices[shardID].searchers.release(s);
+    }
+
+    termStatsCache.put(key, stats);
+    return stats;
+  }
+
+
+  /** Matches docs in the local shard but scores based on
+   *  aggregated stats ("mock distributed scoring") from all
+   *  shards, pull into a shared cache. */ 
+
+  protected class MockSlowShardIndexSearcher extends IndexSearcher {
+    // Version for the sub searchers we search:
+    private final long[] subVersions;
+    private final int shardID;
+
+    public MockSlowShardIndexSearcher(long[] subVersions, IndexReader localReader, int shardID) {
+      super(localReader);
+      this.subVersions = subVersions;
+      this.shardID = shardID;
+
+      // TODO: here we should pre-cache the collection stats
+      // for fields we "know" we will be querying...
+    }
+
+    @Override
+    public TermStatistics termStatistics(Term term, TermContext context) throws IOException {
+      assert term != null;
+      int docFreq = 0;
+      long totalTermFreq = 0;
+      for(int shardID=0;shardID<subVersions.length;shardID++) {
+
+        // nocommit is this right???
+        final TermContext subContext;
+        if (shardID == this.shardID ){
+          subContext = context;
+        } else {
+          subContext = null;
+        }
+        
+        final TermStatistics subStats = getShardTermStats(shardID, subVersions[shardID], term, subContext);
+        docFreq += subStats.docFreq();
+        totalTermFreq += subStats.totalTermFreq();
+      }
+
+      return new TermStatistics(term.bytes(), docFreq, totalTermFreq);
+    }
+
+    @Override
+    public CollectionStatistics collectionStatistics(String field) throws IOException {
+      int docCount = 0;
+      long sumTotalTermFreq = 0;
+      long sumDocFreq = 0;
+      int maxDoc = 0;
+
+      for(int shardID=0;shardID<subVersions.length;shardID++) {
+        final CollectionStatistics subStats = getShardCollectionStats(shardID, subVersions[shardID], field);
+        docCount += subStats.docCount();
+        sumTotalTermFreq += subStats.sumTotalTermFreq();
+        sumDocFreq += subStats.sumDocFreq();
+        maxDoc += subStats.maxDoc();
+      }
+
+      return new CollectionStatistics(field, maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
+    }
+  }
+
+  // TODO: override rewrite and pre-cache all Terms ref'd by
+  // the rewritten Query, ie it should be one trip on the
+  // wire for the full set of terms
+}

Property changes on: lucene/src/test-framework/java/org/apache/lucene/search/ShardSearchingTestBase.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native