Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java =================================================================== --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (revision 1187973) +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (working copy) @@ -408,6 +408,11 @@ } @Override + public long getUniqueTermCount() throws IOException { + return -1; + } + + @Override public Comparator getComparator() { return BytesRef.getUTF8SortedAsUnicodeComparator(); } Index: lucene/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java =================================================================== --- lucene/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java (revision 1187973) +++ lucene/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java (working copy) @@ -139,6 +139,11 @@ public int getDocCount() throws IOException { return -1; } + + @Override + public long getUniqueTermCount() throws IOException { + return -1; + } }); assert termsEnum != null; Index: lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java =================================================================== --- lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java (revision 1187973) +++ lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java (working copy) @@ -203,12 +203,18 @@ // ------------------------ Not implemented methods ------------------------ + @Override public TermsEnum iterator() throws IOException { return null; } @Override + public long getUniqueTermCount() throws IOException { + return -1; + } + + @Override public Comparator getComparator() throws IOException { return null; } Index: lucene/src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReader.java (revision 1187973) +++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -1201,7 +1201,7 @@ // LUCENE-1586: getUniqueTermCount public void testUniqueTermCount() throws Exception { Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); Document doc = new Document(); doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", TextField.TYPE_UNSTORED)); doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", TextField.TYPE_UNSTORED)); Index: lucene/src/java/org/apache/lucene/index/MultiTerms.java =================================================================== --- lucene/src/java/org/apache/lucene/index/MultiTerms.java (revision 1187973) +++ lucene/src/java/org/apache/lucene/index/MultiTerms.java (working copy) @@ -96,6 +96,11 @@ } @Override + public long getUniqueTermCount() throws IOException { + return -1; + } + + @Override public long getSumTotalTermFreq() throws IOException { long sum = 0; for(Terms terms : subs) { Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CheckIndex.java (revision 1187973) +++ lucene/src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -939,20 +939,24 @@ is.search(new TermQuery(new Term(field, lastTerm)), 1); } - - // Test seeking by ord - if (hasOrd && status.termCount-termCountStart > 0) { - long termCount; + + // check unique term count + long termCount = -1; + + if (status.termCount-termCountStart > 0) { try { termCount = fields.terms(field).getUniqueTermCount(); } catch (UnsupportedOperationException uoe) { - termCount = -1; + // not supported } - + if (termCount != -1 && termCount != status.termCount - termCountStart) { throw new RuntimeException("termCount mismatch " + termCount + " vs " + (status.termCount - termCountStart)); } - + } + + // Test seeking by ord + if (hasOrd && status.termCount-termCountStart > 0) { int seekCount = (int) Math.min(10000L, termCount); if (seekCount > 0) { BytesRef[] seekTerms = new BytesRef[seekCount]; @@ -995,6 +999,20 @@ } } + // for most implementations, this is boring (just the sum across all fields) + // but codecs that don't work per-field like preflex actually implement this, + // but don't implement it on Terms, so the check isn't redundant. + long uniqueTermCountAllFields = -1; + try { + uniqueTermCountAllFields = reader.getUniqueTermCount(); + } catch (UnsupportedOperationException e) { + // not supported + } + + if (uniqueTermCountAllFields != -1 && status.termCount != uniqueTermCountAllFields) { + throw new RuntimeException("termCount mismatch " + uniqueTermCountAllFields + " vs " + (status.termCount)); + } + msg("OK [" + status.termCount + " terms; " + status.totFreq + " terms/docs pairs; " + status.totPos + " tokens]"); if (verbose && status.blockTreeStats != null && infoStream != null && status.termCount > 0) { Index: lucene/src/java/org/apache/lucene/index/Terms.java =================================================================== --- lucene/src/java/org/apache/lucene/index/Terms.java (revision 1187973) +++ lucene/src/java/org/apache/lucene/index/Terms.java (working copy) @@ -155,10 +155,12 @@ return termsEnum.docsAndPositions(liveDocs, reuse); } - public long getUniqueTermCount() throws IOException { - throw new UnsupportedOperationException("this reader does not implement getUniqueTermCount()"); - } - + /** Returns the number of terms for this field, or -1 if this + * measure isn't stored by the codec. Note that, just like + * other term measures, this measure does not take deleted + * documents into account. */ + public abstract long getUniqueTermCount() throws IOException; + /** Returns the sum of {@link TermsEnum#totalTermFreq} for * all terms in this field, or -1 if this measure isn't * stored by the codec (or if this fields omits term freq Index: lucene/src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexReader.java (revision 1187973) +++ lucene/src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -1589,26 +1589,17 @@ /** Returns the number of unique terms (across all fields) * in this reader. * - * @throws UnsupportedOperationException if this count + * @return number of unique terms or -1 if this count * cannot be easily determined (eg Multi*Readers). * Instead, you should call {@link * #getSequentialSubReaders} and ask each sub reader for * its unique term count. */ public long getUniqueTermCount() throws IOException { - long numTerms = 0; final Fields fields = fields(); if (fields == null) { return 0; } - FieldsEnum it = fields.iterator(); - while(true) { - String field = it.next(); - if (field == null) { - break; - } - numTerms += fields.terms(field).getUniqueTermCount(); - } - return numTerms; + return fields.getUniqueTermCount(); } /** For IndexReader implementations that use Index: lucene/src/java/org/apache/lucene/index/Fields.java =================================================================== --- lucene/src/java/org/apache/lucene/index/Fields.java (revision 1187973) +++ lucene/src/java/org/apache/lucene/index/Fields.java (working copy) @@ -32,5 +32,31 @@ * null if the field does not exist. */ public abstract Terms terms(String field) throws IOException; + /** Returns the number of terms for all fields, or -1 if this + * measure isn't stored by the codec. Note that, just like + * other term measures, this measure does not take deleted + * documents into account. */ + // TODO: deprecate? + public long getUniqueTermCount() throws IOException { + long numTerms = 0; + FieldsEnum it = iterator(); + while(true) { + String field = it.next(); + if (field == null) { + break; + } + Terms terms = terms(field); + if (terms != null) { + final long termCount = terms.getUniqueTermCount(); + if (termCount == -1) { + return -1; + } + + numTerms += termCount; + } + } + return numTerms; + } + public final static Fields[] EMPTY_ARRAY = new Fields[0]; } Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (revision 1187973) +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (working copy) @@ -162,6 +162,11 @@ return preTerms.get(field); } + @Override + public long getUniqueTermCount() throws IOException { + return getTermsDict().size(); + } + synchronized private TermInfosReader getTermsDict() { if (tis != null) { return tis; @@ -241,6 +246,11 @@ } @Override + public long getUniqueTermCount() throws IOException { + return -1; + } + + @Override public long getSumTotalTermFreq() { return -1; } Index: lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (revision 1187973) +++ lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (working copy) @@ -684,11 +684,13 @@ private final long sumTotalTermFreq; private final long sumDocFreq; private final int docCount; + private final int termCount; private FST fst; private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); private final FieldInfo field; - public TermsReader(FieldInfos fieldInfos, IndexInput in) throws IOException { + public TermsReader(FieldInfos fieldInfos, IndexInput in, int termCount) throws IOException { + this.termCount = termCount; final int fieldNumber = in.readVInt(); field = fieldInfos.fieldInfo(fieldNumber); if (field.indexOptions != IndexOptions.DOCS_ONLY) { @@ -718,6 +720,11 @@ } @Override + public long getUniqueTermCount() throws IOException { + return termCount; + } + + @Override public TermsEnum iterator() { return new FSTTermsEnum(field, fst); } @@ -741,7 +748,7 @@ if (termCount == 0) { break; } - final TermsReader termsReader = new TermsReader(state.fieldInfos, in); + final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount); fields.put(termsReader.field.name, termsReader); } } finally {