Index: src/test/org/apache/lucene/index/TestStressIndexing2.java
===================================================================
--- src/test/org/apache/lucene/index/TestStressIndexing2.java	(revision 926738)
+++ src/test/org/apache/lucene/index/TestStressIndexing2.java	(working copy)
@@ -286,14 +286,20 @@
 
     // create mapping from id2 space to id2 based on idField
     idField = StringHelper.intern(idField);
-    final TermsEnum termsEnum = MultiFields.getFields(r1).terms(idField).iterator();
+    final Fields f1 = MultiFields.getFields(r1);
+    if (f1 == null) {
+      // make sure r2 is empty
+      assertNull(MultiFields.getFields(r2));
+      return;
+    }
+    final TermsEnum termsEnum = f1.terms(idField).iterator();
 
     final Bits delDocs1 = MultiFields.getDeletedDocs(r1);
     final Bits delDocs2 = MultiFields.getDeletedDocs(r2);
     
     Fields fields = MultiFields.getFields(r2);
     if (fields == null) {
-      // make sure r1 is in fract empty (eg has only all
+      // make sure r1 is in fact empty (eg has only all
       // deleted docs):
       DocsEnum docs = null;
       while(termsEnum.next() != null) {
Index: src/test/org/apache/lucene/index/FlexTestUtil.java
===================================================================
--- src/test/org/apache/lucene/index/FlexTestUtil.java	(revision 926738)
+++ src/test/org/apache/lucene/index/FlexTestUtil.java	(working copy)
@@ -110,6 +110,9 @@
 
   private static void testBogusFieldTerms(Random rand, IndexReader r) throws Exception {
     final Fields fields = MultiFields.getFields(r);
+    if (fields == null) {
+      return;
+    }
     for(int i=0;i<10;i++) {
       final String f = "bogus" + rand.nextInt() + "reallybogus";
       Terms terms = fields.terms(f);
Index: src/test/org/apache/lucene/util/packed/TestPackedInts.java
===================================================================
--- src/test/org/apache/lucene/util/packed/TestPackedInts.java	(revision 926738)
+++ src/test/org/apache/lucene/util/packed/TestPackedInts.java	(working copy)
@@ -71,16 +71,28 @@
           w.add(values[i]);
         }
         w.finish();
+        final long fp = out.getFilePointer();
         out.close();
 
         IndexInput in = d.openInput("out.bin");
         PackedInts.Reader r = PackedInts.getReader(in);
+        assertEquals(fp, in.getFilePointer());
         for(int i=0;i<valueCount;i++) {
           assertEquals("index=" + i + " ceil=" + ceil + " valueCount="
                   + valueCount + " nbits=" + nbits + " for "
                   + r.getClass().getSimpleName(), values[i], r.get(i));
         }
         in.close();
+
+        in = d.openInput("out.bin");
+        PackedInts.ReaderIterator r2 = PackedInts.getReaderIterator(in);
+        for(int i=0;i<valueCount;i++) {
+          assertEquals("index=" + i + " ceil=" + ceil + " valueCount="
+                  + valueCount + " nbits=" + nbits + " for "
+                  + r.getClass().getSimpleName(), values[i], r2.next());
+        }
+        assertEquals(fp, in.getFilePointer());
+        in.close();
         ceil *= 2;
       }
     }
@@ -193,4 +205,21 @@
       }
     }
   }
+
+  public void testSingleValue() throws Exception {
+    Directory dir = new MockRAMDirectory();
+    IndexOutput out = dir.createOutput("out");
+    PackedInts.Writer w = PackedInts.getWriter(out, 1, 8);
+    w.add(17);
+    w.finish();
+    final long end = out.getFilePointer();
+    out.close();
+
+    IndexInput in = dir.openInput("out");
+    PackedInts.Reader r = PackedInts.getReader(in);
+    assertEquals(end, in.getFilePointer());
+    in.close();
+
+    dir.close();
+  }
 }
Index: src/java/org/apache/lucene/index/codecs/TermsConsumer.java
===================================================================
--- src/java/org/apache/lucene/index/codecs/TermsConsumer.java	(revision 926738)
+++ src/java/org/apache/lucene/index/codecs/TermsConsumer.java	(working copy)
@@ -93,5 +93,7 @@
         }
       }
     }
+
+    finish();
   }
 }
Index: src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
===================================================================
--- src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java	(revision 926738)
+++ src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java	(working copy)
@@ -28,6 +28,7 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.packed.PackedInts;
 
 import java.util.HashMap;
 import java.util.Iterator;
@@ -63,7 +64,7 @@
 /** @lucene.experimental */
 public class SimpleStandardTermsIndexReader extends StandardTermsIndexReader {
 
-  final private int totalIndexInterval;
+  private int totalIndexInterval;
   private int indexDivisor;
   final private int indexInterval;
 
@@ -72,7 +73,13 @@
   private volatile boolean indexLoaded;
 
   private final Comparator<BytesRef> termComp;
+  private final String segment;
 
+  private final static int PAGED_BYTES_BITS = 15;
+
+  // all fields share this single logical byte[]
+  private final PagedBytes termBytes = new PagedBytes(PAGED_BYTES_BITS);
+
   final HashMap<FieldInfo,FieldIndexReader> fields = new HashMap<FieldInfo,FieldIndexReader>();
 
   public SimpleStandardTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator<BytesRef> termComp)
@@ -80,6 +87,8 @@
 
     this.termComp = termComp;
 
+    this.segment = segment;
+
     IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, StandardCodec.TERMS_INDEX_EXTENSION));
     
     boolean success = false;
@@ -118,10 +127,14 @@
           System.out.println("  read field number=" + field);
         }
         final int numIndexTerms = in.readInt();
+        final long termsStart = in.readLong();
         final long indexStart = in.readLong();
+        final long packedIndexStart = in.readLong();
+        final long packedOffsetsStart = in.readLong();
+        assert packedIndexStart >= indexStart: "packedStart=" + packedIndexStart + " indexStart=" + indexStart + " numIndexTerms=" + numIndexTerms + " seg=" + segment;
         if (numIndexTerms > 0) {
           final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
-          fields.put(fieldInfo, new FieldIndexReader(in, fieldInfo, numIndexTerms, indexStart));
+          fields.put(fieldInfo, new FieldIndexReader(in, fieldInfo, numIndexTerms, indexStart, termsStart, packedIndexStart, packedOffsetsStart));
         }
       }
       success = true;
@@ -130,57 +143,15 @@
         in.close();
         this.in = null;
         if (success) {
-          trimByteBlock();
           indexLoaded = true;
         }
+        termBytes.finish();
       } else {
         this.in = in;
       }
     }
   }
 
-  /* Called when index is fully loaded.  We know we will use
-   * no more bytes in the final byte[], so trim it down to
-   * its actual usagee.  This substantially reduces memory
-   * usage of SegmentReader searching a tiny segment. */
-  private final void trimByteBlock() {
-    if (blockOffset == 0) {
-      // There were no fields in this segment:
-      if (blocks != null) {
-        blocks[blockUpto] = null;
-      }
-    } else {
-      byte[] last = new byte[blockOffset];
-      System.arraycopy(blocks[blockUpto], 0, last, 0, blockOffset);
-      blocks[blockUpto] = last;
-    }
-  }
-
-  // TODO: we can record precisely how many bytes are
-  // required during indexing, save that into file, and be
-  // precise when we allocate the blocks; we even don't need
-  // to use blocks anymore (though my still want to, to
-  // prevent allocation failure due to mem fragmentation on
-  // 32bit)
-
-  // Fixed size byte blocks, to hold all term bytes; these
-  // blocks are shared across fields
-  private byte[][] blocks;
-  int blockUpto;
-  int blockOffset;
-
-  private static final int BYTE_BLOCK_SHIFT = 15;
-  private static final int BYTE_BLOCK_SIZE = 1 << BYTE_BLOCK_SHIFT;
-  private static final int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1;
-
-  static {
-    // Make sure DW can't ever write a term whose length
-    // cannot be encoded with short (because we use short[]
-    // to hold the length of each term).
-    assert IndexWriter.MAX_TERM_LENGTH < Short.MAX_VALUE;
-    assert BYTE_BLOCK_SIZE >= IndexWriter.MAX_TERM_LENGTH;
-  }
-
   private final class FieldIndexReader extends FieldReader {
 
     final private FieldInfo fieldInfo;
@@ -190,14 +161,21 @@
     private final IndexInput in;
 
     private final long indexStart;
+    private final long termsStart;
+    private final long packedIndexStart;
+    private final long packedOffsetsStart;
 
     private final int numIndexTerms;
 
-    public FieldIndexReader(IndexInput in, FieldInfo fieldInfo, int numIndexTerms, long indexStart) throws IOException {
+    public FieldIndexReader(IndexInput in, FieldInfo fieldInfo, int numIndexTerms, long indexStart, long termsStart, long packedIndexStart,
+                            long packedOffsetsStart) throws IOException {
 
       this.fieldInfo = fieldInfo;
       this.in = in;
+      this.termsStart = termsStart;
       this.indexStart = indexStart;
+      this.packedIndexStart = packedIndexStart;
+      this.packedOffsetsStart = packedOffsetsStart;
       this.numIndexTerms = numIndexTerms;
 
       // We still create the indexReader when indexDivisor
@@ -210,6 +188,9 @@
         }
 
         coreIndex = new CoreFieldIndex(indexStart,
+                                       termsStart,
+                                       packedIndexStart,
+                                       packedOffsetsStart,
                                        numIndexTerms);
       
       } else {
@@ -221,7 +202,7 @@
 
     public void loadTermsIndex() throws IOException {
       if (coreIndex == null) {
-        coreIndex = new CoreFieldIndex(indexStart, numIndexTerms);
+        coreIndex = new CoreFieldIndex(indexStart, termsStart, packedIndexStart, packedOffsetsStart, numIndexTerms);
       }
     }
 
@@ -263,150 +244,115 @@
 
     private final class CoreFieldIndex {
 
-      // TODO: used packed ints here
-      // Pointer into terms dict file that we are indexing
-      final long[] fileOffset;
+      final private long termBytesStart;
 
-      // TODO: used packed ints here
-      // For each term, points to start of term's bytes within
-      // block.
-      // TODO: wasteful that this is always long; many terms
-      // dict indexes obviously don't require so much address
-      // space; since we know up front during indexing how
-      // much space is needed we could pack this to the
-      // precise # bits
-      final long[] blockPointer;
-    
-      // TODO: used packed ints here: we know max term
-      // length; often its small
+      // offset into index termBytes
+      final PackedInts.Reader termOffsets;
 
-      // TODO: can we inline this w/ the bytes?  like
-      // DW.  vast majority of terms only need 1 byte, not 2
+      // index pointers into main terms dict
+      final PackedInts.Reader termsDictOffsets;
 
-      // Length of each term
-      final short[] termLength;
-
       final int numIndexTerms;
 
-      CoreFieldIndex(long indexStart, int numIndexTerms) throws IOException {
+      final long termsStart;
 
+      public CoreFieldIndex(long indexStart, long termsStart, long packedIndexStart, long packedOffsetsStart, int numIndexTerms) throws IOException {
+
+        this.termsStart = termsStart;
+        termBytesStart = termBytes.getPointer();
+
         IndexInput clone = (IndexInput) in.clone();
         clone.seek(indexStart);
 
-        if (indexDivisor == -1) {
-          // Special case: we are being loaded inside
-          // IndexWriter because a SegmentReader that at
-          // first was opened for merging, is now being
-          // opened to perform deletes or for an NRT reader
-          this.numIndexTerms = numIndexTerms;
-        } else {
-          this.numIndexTerms = 1+(numIndexTerms-1) / indexDivisor;
-        }
+        // -1 is passed to mean "don't load term index", but
+        // if we are then later loaded it's overwritten with
+        // a real value
+        assert indexDivisor > 0;
 
+        this.numIndexTerms = 1+(numIndexTerms-1) / indexDivisor;
+
         assert this.numIndexTerms  > 0: "numIndexTerms=" + numIndexTerms + " indexDivisor=" + indexDivisor;
 
-        if (blocks == null) {
-          blocks = new byte[1][];
-          blocks[0] = new byte[BYTE_BLOCK_SIZE];
-        }
+        if (indexDivisor == 1) {
+          // Default (load all index terms) is fast -- slurp in the images from disk:
+          
+          try {
+            final long numTermBytes = packedIndexStart - indexStart;
+            termBytes.copy(clone, numTermBytes);
 
-        byte[] lastBlock = blocks[blockUpto];
-        int lastBlockOffset = blockOffset;
+            // records offsets into main terms dict file
+            termsDictOffsets = PackedInts.getReader(clone);
+            assert termsDictOffsets.size() == numIndexTerms;
 
-        fileOffset = new long[this.numIndexTerms];
-        blockPointer = new long[this.numIndexTerms];
-        termLength = new short[this.numIndexTerms];
-        
-        final byte[] skipBytes;
-        if (indexDivisor != 1) {
-          // only need skipBytes (below) if we are not
-          // loading all index terms
-          skipBytes = new byte[128];
+            // records offsets into byte[] term data
+            termOffsets = PackedInts.getReader(clone);
+            assert termOffsets.size() == 1+numIndexTerms;
+          } finally {
+            clone.close();
+          }
         } else {
-          skipBytes = null;
-        }
+          // Get packed iterators
+          final IndexInput clone1 = (IndexInput) in.clone();
+          final IndexInput clone2 = (IndexInput) in.clone();
 
-        int upto = 0;
-        long pointer = 0;
-      
-        for(int i=0;i<numIndexTerms;i++) {
-          final int start = clone.readVInt();
-          final int suffix = clone.readVInt();
-          final int thisTermLength = start + suffix;
+          try {
+            // Subsample the index terms
+            clone1.seek(packedIndexStart);
+            final PackedInts.ReaderIterator termsDictOffsetsIter = PackedInts.getReaderIterator(clone1);
 
-          assert thisTermLength <= BYTE_BLOCK_SIZE;
+            clone2.seek(packedOffsetsStart);
+            final PackedInts.ReaderIterator termOffsetsIter = PackedInts.getReaderIterator(clone2);
 
-          if (i%indexDivisor == 0) {
-            // Keeper
-            if (blockOffset + thisTermLength > BYTE_BLOCK_SIZE) {
-              // New block
-              final byte[] newBlock = new byte[BYTE_BLOCK_SIZE];
-              if (blocks.length == blockUpto+1) {
-                final int newSize = ArrayUtil.oversize(blockUpto+2, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
-                final byte[][] newBlocks = new byte[newSize][];
-                System.arraycopy(blocks, 0, newBlocks, 0, blocks.length);
-                blocks = newBlocks;
-              }
-              blockUpto++;
-              blocks[blockUpto] = newBlock;
-              blockOffset = 0;
-            }
+            // TODO: often we can get by w/ fewer bits per
+            // value, below.. .but this'd be more complex:
+            // we'd have to try @ fewer bits and then grow
+            // if we overflowed it.
 
-            final byte[] block = blocks[blockUpto];
+            PackedInts.Mutable termsDictOffsetsM = PackedInts.getMutable(this.numIndexTerms, termsDictOffsetsIter.getBitsPerValue());
+            PackedInts.Mutable termOffsetsM = PackedInts.getMutable(this.numIndexTerms+1, termOffsetsIter.getBitsPerValue());
 
-            // Copy old prefix
-            assert lastBlock != null || start == 0;
-            assert block != null;
-            System.arraycopy(lastBlock, lastBlockOffset, block, blockOffset, start);
+            termsDictOffsets = termsDictOffsetsM;
+            termOffsets = termOffsetsM;
 
-            // Read new suffix
-            clone.readBytes(block, blockOffset+start, suffix);
+            int upto = 0;
 
-            // Advance file offset
-            pointer += clone.readVLong();
+            long lastTermOffset = 0;
+            long termOffsetUpto = 0;
 
-            assert thisTermLength < Short.MAX_VALUE;
+            while(upto < this.numIndexTerms) {
+              // main file offset copies straight over
+              termsDictOffsetsM.set(upto, termsDictOffsetsIter.next());
 
-            termLength[upto] = (short) thisTermLength;
-            fileOffset[upto] = pointer;
-            blockPointer[upto] = blockUpto * BYTE_BLOCK_SIZE + blockOffset;
+              termOffsetsM.set(upto, termOffsetUpto);
+              upto++;
 
-            /*
-            BytesRef tr = new BytesRef();
-            tr.bytes = blocks[blockUpto];
-            tr.offset = blockOffset;
-            tr.length = thisTermLength;
+              long termOffset = termOffsetsIter.next();
+              long nextTermOffset = termOffsetsIter.next();
+              final int numTermBytes = (int) (nextTermOffset - termOffset);
 
-            //System.out.println("    read index term=" + new String(blocks[blockUpto], blockOffset, thisTermLength, "UTF-8") + " this=" + this + " bytes=" + block + " (vs=" + blocks[blockUpto] + ") offset=" + blockOffset);
-            //System.out.println("    read index term=" + tr.toBytesString() + " this=" + this + " bytes=" + block + " (vs=" + blocks[blockUpto] + ") offset=" + blockOffset);
-            */
+              clone.seek(indexStart + termOffset);
+              assert indexStart + termOffset < clone.length() : "indexStart=" + indexStart + " termOffset=" + termOffset + " len=" + clone.length();
+              assert indexStart + termOffset + numTermBytes < clone.length();
 
-            lastBlock = block;
-            lastBlockOffset = blockOffset;
-            blockOffset += thisTermLength;
-            upto++;
-          } else {
-            // Skip bytes
-            int toSkip = suffix;
-            while(true) {
-              if (toSkip > skipBytes.length) {
-                clone.readBytes(skipBytes, 0, skipBytes.length);
-                toSkip -= skipBytes.length;
-              } else {
-                clone.readBytes(skipBytes, 0, toSkip);
-                break;
+              termBytes.copy(clone, numTermBytes);
+              termOffsetUpto += numTermBytes;
+
+              // skip terms:
+              termsDictOffsetsIter.next();
+              for(int i=0;i<indexDivisor-2;i++) {
+                termOffsetsIter.next();
+                termsDictOffsetsIter.next();
               }
             }
+            termOffsetsM.set(upto, termOffsetUpto);
 
-            // Advance file offset
-            pointer += clone.readVLong();
+          } finally {
+            clone1.close();
+            clone2.close();
+            clone.close();
           }
         }
 
-        clone.close();
-
-        assert upto == this.numIndexTerms;
-
         if (Codec.DEBUG) {
           System.out.println("  done read");
         }
@@ -423,30 +369,28 @@
       }
 
       private final void fillResult(int idx, TermsIndexResult result) {
-        final long loc = blockPointer[idx];
-        result.term.bytes = blocks[(int) (loc >> BYTE_BLOCK_SHIFT)];
-        result.term.offset = (int) (loc & BYTE_BLOCK_MASK);
-        result.term.length = termLength[idx];
+        final long offset = termOffsets.get(idx);
+        final int length = (int) (termOffsets.get(1+idx) - offset);
+        termBytes.fill(result.term, termBytesStart + offset, length);
         result.position = idx * totalIndexInterval;
-        result.offset = fileOffset[idx];
+        result.offset = termsStart + termsDictOffsets.get(idx);
       }
 
       public final void getIndexOffset(BytesRef term, TermsIndexResult result) throws IOException {
 
         if (Codec.DEBUG) {
-          System.out.println("getIndexOffset field=" + fieldInfo.name + " term=" + term + " indexLen = " + blockPointer.length + " numIndexTerms=" + fileOffset.length + " numIndexedTerms=" + fileOffset.length);
+          System.out.println("getIndexOffset field=" + fieldInfo.name + " term=" + term.utf8ToString());
         }
 
         int lo = 0;					  // binary search
-        int hi = fileOffset.length - 1;
+        int hi = numIndexTerms - 1;
 
         while (hi >= lo) {
           int mid = (lo + hi) >>> 1;
 
-          final long loc = blockPointer[mid];
-          result.term.bytes = blocks[(int) (loc >> BYTE_BLOCK_SHIFT)];
-          result.term.offset = (int) (loc & BYTE_BLOCK_MASK);
-          result.term.length = termLength[mid];
+          final long offset = termOffsets.get(mid);
+          final int length = (int) (termOffsets.get(1+mid) - offset);
+          termBytes.fill(result.term, termBytesStart + offset, length);
 
           int delta = termComp.compare(term, result.term);
           if (delta < 0) {
@@ -456,7 +400,7 @@
           } else {
             assert mid >= 0;
             result.position = mid*totalIndexInterval;
-            result.offset = fileOffset[mid];
+            result.offset = termsStart + termsDictOffsets.get(mid);
             return;
           }
         }
@@ -465,13 +409,12 @@
           hi = 0;
         }
 
-        final long loc = blockPointer[hi];
-        result.term.bytes = blocks[(int) (loc >> BYTE_BLOCK_SHIFT)];
-        result.term.offset = (int) (loc & BYTE_BLOCK_MASK);
-        result.term.length = termLength[hi];
+        final long offset = termOffsets.get(hi);
+        final int length = (int) (termOffsets.get(1+hi) - offset);
+        termBytes.fill(result.term, termBytesStart + offset, length);
 
         result.position = hi*totalIndexInterval;
-        result.offset = fileOffset[hi];
+        result.offset = termsStart + termsDictOffsets.get(hi);
       }
 
       public final void getIndexOffset(long ord, TermsIndexResult result) throws IOException {
@@ -488,6 +431,7 @@
     if (!indexLoaded) {
 
       this.indexDivisor = indexDivisor;
+      this.totalIndexInterval = indexInterval * indexDivisor;
 
       // mxx
       if (Codec.DEBUG) {
@@ -498,10 +442,10 @@
       while(it.hasNext()) {
         it.next().loadTermsIndex();
       }
-      trimByteBlock();
 
       indexLoaded = true;
       in.close();
+      termBytes.finish();
     }
   }
 
Index: src/java/org/apache/lucene/index/codecs/standard/StandardTermsIndexWriter.java
===================================================================
--- src/java/org/apache/lucene/index/codecs/standard/StandardTermsIndexWriter.java	(revision 926738)
+++ src/java/org/apache/lucene/index/codecs/standard/StandardTermsIndexWriter.java	(working copy)
@@ -29,6 +29,7 @@
 
   public abstract class FieldWriter {
     public abstract boolean checkIndexTerm(BytesRef text, int docFreq) throws IOException;
+    public abstract void finish() throws IOException;
   }
 
   public abstract FieldWriter addField(FieldInfo fieldInfo);
Index: src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
===================================================================
--- src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java	(revision 926738)
+++ src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java	(working copy)
@@ -372,7 +372,7 @@
           indexReader.getIndexOffset(term, indexResult);
 
           if (Codec.DEBUG) {
-            Codec.debug(" index pos=" + indexResult.position + " termFP=" + indexResult.offset + " term=" + indexResult.term + " this=" + this);
+            Codec.debug(" index pos=" + indexResult.position + " termFP=" + indexResult.offset + " term=" + indexResult.term.utf8ToString() + " this=" + this);
           }
 
           in.seek(indexResult.offset);
@@ -507,6 +507,9 @@
         }
         
         if (state.ord >= numTerms-1) {
+          if (Codec.DEBUG) {
+            Codec.debug("  return null ord=" + state.ord + " vs numTerms-1=" + (numTerms-1));
+          }
           return null;
         }
 
@@ -514,7 +517,7 @@
         state.docFreq = in.readVInt();
 
         if (Codec.DEBUG) {
-          Codec.debug("  text=" + bytesReader.term + " freq=" + state.docFreq + " tis=" + in);
+          Codec.debug("  text=" + bytesReader.term.utf8ToString() + " freq=" + state.docFreq + " tis=" + in);
         }
 
         // TODO: would be cleaner, but space-wasting, to
Index: src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexWriter.java
===================================================================
--- src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexWriter.java	(revision 926738)
+++ src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexWriter.java	(working copy)
@@ -25,6 +25,8 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.packed.PackedInts;
 
 import java.util.List;
 import java.util.ArrayList;
@@ -58,7 +60,6 @@
     // Placeholder for dir offset
     out.writeLong(0);
     out.writeInt(termIndexInterval);
-    termWriter = new DeltaBytesWriter(out);
   }
 
   @Override
@@ -66,8 +67,6 @@
     this.termsOut = termsOut;
   }
   
-  final private DeltaBytesWriter termWriter;
-
   @Override
   public FieldWriter addField(FieldInfo field) {
     SimpleFieldWriter writer = new SimpleFieldWriter(field);
@@ -78,33 +77,99 @@
   private class SimpleFieldWriter extends FieldWriter {
     final FieldInfo fieldInfo;
     int numIndexTerms;
-    private long lastTermsPointer;
     final long indexStart;
+    final long termsStart;
+    long packedIndexStart;
+    long packedOffsetsStart;
     private int numTerms;
 
+    // TODO: we could conceivably make a PackedInts wrapper
+    // that auto-grows... then we wouldn't force 6 bytes RAM
+    // per index term:
+    private short[] termLengths;
+    private int[] termsPointerDeltas;
+    private long lastTermsPointer;
+    private long totTermLength;
+
     SimpleFieldWriter(FieldInfo fieldInfo) {
       this.fieldInfo = fieldInfo;
       indexStart = out.getFilePointer();
-      termWriter.reset();
+      termsStart = lastTermsPointer = termsOut.getFilePointer();
+      termLengths = new short[0];
+      termsPointerDeltas = new int[0];
     }
 
     @Override
     public boolean checkIndexTerm(BytesRef text, int docFreq) throws IOException {
       // First term is first indexed term:
       if (0 == (numTerms++ % termIndexInterval)) {
-        final long termsPointer = termsOut.getFilePointer();
+
         if (Codec.DEBUG) {
-          Codec.debug("sstiw.checkIndexTerm write index field=" + fieldInfo.name + " term=" + text + " termsFP=" + termsPointer + " numIndexTerms=" + numIndexTerms + " outFP=" + out.getFilePointer());
+          Codec.debug("sstiw.checkIndexTerm write index field=" + fieldInfo.name + " term=" + text.utf8ToString() + " numIndexTerms=" + numIndexTerms + " outFP=" + out.getFilePointer());
         }
-        termWriter.write(text);
-        out.writeVLong(termsPointer - lastTermsPointer);
-        lastTermsPointer = termsPointer;
+
+        // write full bytes
+        out.writeBytes(text.bytes, text.offset, text.length);
+
+        if (termLengths.length == numIndexTerms) {
+          termLengths = ArrayUtil.grow(termLengths);
+        }
+        if (termsPointerDeltas.length == numIndexTerms) {
+          termsPointerDeltas = ArrayUtil.grow(termsPointerDeltas);
+        }
+
+        // save delta terms pointer
+        final long fp = termsOut.getFilePointer();
+        termsPointerDeltas[numIndexTerms] = (int) (fp - lastTermsPointer);
+        lastTermsPointer = fp;
+
+        // save term length (in bytes)
+        assert text.length <= Short.MAX_VALUE;
+        termLengths[numIndexTerms] = (short) text.length;
+
+        totTermLength += text.length;
+
         numIndexTerms++;
         return true;
       } else {
         return false;
       }
     }
+
+    @Override
+    public void finish() throws IOException {
+
+      // write primary terms dict offsets
+      packedIndexStart = out.getFilePointer();
+
+      final long maxValue = termsOut.getFilePointer();
+      PackedInts.Writer w = PackedInts.getWriter(out, numIndexTerms, PackedInts.bitsRequired(maxValue));
+
+      // relative to our indexStart
+      long upto = 0;
+      for(int i=0;i<numIndexTerms;i++) {
+        upto += termsPointerDeltas[i];
+        w.add(upto);
+      }
+      w.finish();
+
+      packedOffsetsStart = out.getFilePointer();
+
+      // write offsets into the byte[] terms
+      w = PackedInts.getWriter(out, 1+numIndexTerms, PackedInts.bitsRequired(totTermLength));
+      upto = 0;
+      for(int i=0;i<numIndexTerms;i++) {
+        w.add(upto);
+        upto += termLengths[i];
+      }
+      w.add(upto);
+      w.finish();
+
+      // our referrer holds onto us, while other fields are
+      // being written, so don't tie up this RAM:
+      termLengths = null;
+      termsPointerDeltas = null;
+    }
   }
 
   @Override
@@ -123,7 +188,10 @@
       }
       out.writeInt(field.fieldInfo.number);
       out.writeInt(field.numIndexTerms);
+      out.writeLong(field.termsStart);
       out.writeLong(field.indexStart);
+      out.writeLong(field.packedIndexStart);
+      out.writeLong(field.packedOffsetsStart);
     }
     out.seek(CodecUtil.headerLength(CODEC_NAME));
     out.writeLong(dirStart);
Index: src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java
===================================================================
--- src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java	(revision 926738)
+++ src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java	(working copy)
@@ -205,7 +205,8 @@
 
     // Finishes all terms in this field
     @Override
-    public void finish() {
+    public void finish() throws IOException {
+      fieldIndexWriter.finish();
     }
   }
-}
\ No newline at end of file
+}
Index: src/java/org/apache/lucene/index/codecs/standard/PagedBytes.java
===================================================================
--- src/java/org/apache/lucene/index/codecs/standard/PagedBytes.java	(revision 0)
+++ src/java/org/apache/lucene/index/codecs/standard/PagedBytes.java	(revision 0)
@@ -0,0 +1,129 @@
+package org.apache.lucene.index.codecs.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.CloseableThreadLocal;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.store.IndexInput;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.io.Closeable;
+import java.io.IOException;
+
+/** Represents a logical byte[] as a series of pages.  You
+ *  can write-once into the logical byte[], using copy, and
+ *  then retrieve slices (BytesRef) into it using fill. */
+class PagedBytes implements Closeable {
+  private final List<byte[]> blocks = new ArrayList<byte[]>();
+  private final int blockSize;
+  private final int blockBits;
+  private final int blockMask;
+  private int upto;
+  private byte[] currentBlock;
+  private final CloseableThreadLocal<byte[]> threadBuffers = new CloseableThreadLocal();
+
+  private static final byte[] EMPTY_BYTES = new byte[0];
+
+  /** 1<<blockBits must be bigger than biggest single
+   *  BytesRef slice that will be pulled */
+  public PagedBytes(int blockBits) {
+    this.blockSize = 1 << blockBits;
+    this.blockBits = blockBits;
+    blockMask = blockSize-1;
+    upto = blockSize;
+  }
+
+  /** Read this many bytes from in */
+  public void copy(IndexInput in, long byteCount) throws IOException {
+    while (byteCount > 0) {
+      int left = blockSize - upto;
+      if (left == 0) {
+        if (currentBlock != null) {
+          blocks.add(currentBlock);
+        }
+        currentBlock = new byte[blockSize];
+        upto = 0;
+        left = blockSize;
+      }
+      if (left < byteCount) {
+        in.readBytes(currentBlock, upto, left, false);
+        upto = blockSize;
+        byteCount -= left;
+      } else {
+        in.readBytes(currentBlock, upto, (int) byteCount, false);
+        upto += byteCount;
+        byteCount = 0;
+      }
+    }
+  }
+
+  /** Commits final byte[], trimming it if necessary. */
+  public void finish() {
+    if (upto < blockSize) {
+      final byte[] newBlock = new byte[upto];
+      System.arraycopy(currentBlock, 0, newBlock, 0, upto);
+      currentBlock = newBlock;
+    }
+    if (currentBlock == null) {
+      currentBlock = EMPTY_BYTES;
+    }
+    blocks.add(currentBlock);
+    currentBlock = null;
+  }
+
+  public long getPointer() {
+    if (currentBlock == null) {
+      return 0;
+    } else {
+      return (blocks.size() * ((long) blockSize)) + upto;
+    }
+  }
+
+  /** Get a slice out of the byte array. */
+  public void fill(BytesRef b, long start, int length) {
+    assert length >= 0: "length=" + length;
+    final int index = (int) (start >> blockBits);
+    final int offset = (int) (start & blockMask);
+    b.length = length;
+    if (blockSize - offset >= length) {
+      // Within block
+      b.bytes = blocks.get(index);
+      b.offset = offset;
+    } else {
+      // Split
+      byte[] buffer = threadBuffers.get();
+      if (buffer == null) {
+        buffer = new byte[length];
+        threadBuffers.set(buffer);
+      } else if (buffer.length < length) {
+        buffer = ArrayUtil.grow(buffer, length);
+        threadBuffers.set(buffer);
+      }
+      b.bytes = buffer;
+      b.offset = 0;
+      System.arraycopy(blocks.get(index), offset, buffer, 0, blockSize-offset);
+      System.arraycopy(blocks.get(1+index), 0, buffer, blockSize-offset, length-(blockSize-offset));
+    }
+  }
+
+  public void close() {
+    threadBuffers.close();
+  }
+}

Property changes on: src/java/org/apache/lucene/index/codecs/standard/PagedBytes.java
___________________________________________________________________
Added: svn:eol-style
   + native

Index: src/java/org/apache/lucene/util/packed/Packed64.java
===================================================================
--- src/java/org/apache/lucene/util/packed/Packed64.java	(revision 926738)
+++ src/java/org/apache/lucene/util/packed/Packed64.java	(working copy)
@@ -148,6 +148,7 @@
     super(valueCount, bitsPerValue);
     int size = size(valueCount, bitsPerValue);
     blocks = new long[size+1]; // +1 due to non-conditional tricks
+    // TODO: find a faster way to bulk-read longs...
     for(int i=0;i<size;i++) {
       blocks[i] = in.readLong();
     }
Index: src/java/org/apache/lucene/util/packed/PackedWriter.java
===================================================================
--- src/java/org/apache/lucene/util/packed/PackedWriter.java	(revision 926738)
+++ src/java/org/apache/lucene/util/packed/PackedWriter.java	(working copy)
@@ -45,7 +45,7 @@
     pendingBitPos = 64;
     masks = new long[bitsPerValue - 1];
 
-    int v = 1;
+    long v = 1;
     for (int i = 0; i < bitsPerValue - 1; i++) {
       v *= 2;
       masks[i] = v - 1;
@@ -104,7 +104,6 @@
     if (pendingBitPos != 64) {
       out.writeLong(pending);
     }
-    out.writeLong(0L); // Dummy to compensate for not using conditionals
   }
 
   public String toString() {
Index: src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
===================================================================
--- src/java/org/apache/lucene/util/packed/PackedReaderIterator.java	(revision 0)
+++ src/java/org/apache/lucene/util/packed/PackedReaderIterator.java	(revision 0)
@@ -0,0 +1,84 @@
+package org.apache.lucene.util.packed;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.IndexInput;
+
+import java.io.IOException;
+
+class PackedReaderIterator implements PackedInts.ReaderIterator {
+  private long pending;
+  private int pendingBitsLeft;
+  private final IndexInput in;
+  private final int bitsPerValue;
+  private final int valueCount;
+
+  // masks[n-1] masks for bottom n bits
+  private final long[] masks;
+
+  public PackedReaderIterator(int bitsPerValue, int valueCount, IndexInput in)
+    throws IOException {
+
+    this.valueCount = valueCount;
+    this.bitsPerValue = bitsPerValue;
+    
+    this.in = in;
+
+    masks = new long[bitsPerValue];
+
+    long v = 1;
+    for (int i = 0; i < bitsPerValue; i++) {
+      v *= 2;
+      masks[i] = v - 1;
+    }
+  }
+
+  public int getBitsPerValue() {
+    return bitsPerValue;
+  }
+
+  public int size() {
+    return valueCount;
+  }
+
+  public long next() throws IOException {
+    if (pendingBitsLeft == 0) {
+      pending = in.readLong();
+      pendingBitsLeft = 64;
+    }
+
+    if (pendingBitsLeft >= bitsPerValue) {
+      // not split
+      final long result = (pending >> (pendingBitsLeft - bitsPerValue)) & masks[bitsPerValue-1];
+      pendingBitsLeft -= bitsPerValue;
+      return result;
+    } else {
+      // split
+      final int bits1 = bitsPerValue - pendingBitsLeft;
+      final long result1 = (pending & masks[pendingBitsLeft-1]) << bits1;
+      pending = in.readLong();
+      final long result2 = (pending >> (64 - bits1)) & masks[bits1-1];
+      pendingBitsLeft = 64 + pendingBitsLeft - bitsPerValue;
+      return result1 | result2;
+    }
+  }
+
+  public void close() throws IOException {
+    in.close();
+  }
+}

Property changes on: src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
___________________________________________________________________
Added: svn:eol-style
   + native

Index: src/java/org/apache/lucene/util/packed/PackedInts.java
===================================================================
--- src/java/org/apache/lucene/util/packed/PackedInts.java	(revision 926738)
+++ src/java/org/apache/lucene/util/packed/PackedInts.java	(working copy)
@@ -17,6 +17,8 @@
  * limitations under the License.
  */
 
+import java.io.Closeable;
+
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.CodecUtil;
@@ -65,6 +67,18 @@
   }
 
   /**
+   * Run-once iterator interface, to decode previously saved PackedInts.
+   */
+  public static interface ReaderIterator extends Closeable {
+    /** Returns next value */
+    long next() throws IOException;
+    /** Returns number of bits per value */
+    int getBitsPerValue();
+    /** Returns number of values */
+    int size();
+  }
+
+  /**
    * A packed integer array that can be modified.
    * @lucene.internal
    */
@@ -167,6 +181,22 @@
   }
 
   /**
+   * Retrieve PackedInts as a {@link ReaderIterator}
+   * @param in positioned at the beginning of a stored packed int structure.
+   * @return an iterator to access the values
+   * @throws IOException if the structure could not be retrieved.
+   * @lucene.internal
+   */
+  public static ReaderIterator getReaderIterator(IndexInput in) throws IOException {
+    CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START);
+    final int bitsPerValue = in.readVInt();
+    assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
+    final int valueCount = in.readVInt();
+
+    return new PackedReaderIterator(bitsPerValue, valueCount, in);
+  }
+
+  /**
    * Create a packed integer array with the given amount of values initialized
    * to 0. the valueCount and the bitsPerValue cannot be changed after creation.
    * All Mutables known by this factory are kept fully in RAM.
@@ -228,7 +258,7 @@
     } if (maxValue > 0x1FFFFFFFFFFFFFFFL) {
       return 62;
     }
-    return (int) Math.ceil(Math.log(1+maxValue)/Math.log(2.0));
+    return Math.max(1, (int) Math.ceil(Math.log(1+maxValue)/Math.log(2.0)));
   }
 
   /**
Index: src/java/org/apache/lucene/util/packed/Packed32.java
===================================================================
--- src/java/org/apache/lucene/util/packed/Packed32.java	(revision 926738)
+++ src/java/org/apache/lucene/util/packed/Packed32.java	(working copy)
@@ -129,6 +129,7 @@
     super(valueCount, bitsPerValue);
     int size = size(bitsPerValue, valueCount);
     blocks = new int[size + 1]; // +1 due to non-conditional tricks
+    // TODO: find a faster way to bulk-read ints...
     for(int i = 0 ; i < size ; i++) {
       blocks[i] = in.readInt();
     }
Index: src/java/org/apache/lucene/util/ArrayUtil.java
===================================================================
--- src/java/org/apache/lucene/util/ArrayUtil.java	(revision 926738)
+++ src/java/org/apache/lucene/util/ArrayUtil.java	(working copy)
@@ -232,6 +232,29 @@
       return currentSize;
   }
 
+  public static short[] grow(short[] array, int minSize) {
+    if (array.length < minSize) {
+      short[] newArray = new short[oversize(minSize, RamUsageEstimator.NUM_BYTES_SHORT)];
+      System.arraycopy(array, 0, newArray, 0, array.length);
+      return newArray;
+    } else
+      return array;
+  }
+
+  public static short[] grow(short[] array) {
+    return grow(array, 1 + array.length);
+  }
+
+  public static short[] shrink(short[] array, int targetSize) {
+    final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_SHORT);
+    if (newSize != array.length) {
+      short[] newArray = new short[newSize];
+      System.arraycopy(array, 0, newArray, 0, newSize);
+      return newArray;
+    } else
+      return array;
+  }
+
   public static int[] grow(int[] array, int minSize) {
     if (array.length < minSize) {
       int[] newArray = new int[oversize(minSize, RamUsageEstimator.NUM_BYTES_INT)];