Index: lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java (revision 1430364) +++ lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java (working copy) @@ -287,7 +287,8 @@ allowRandomSuffixSharing ? _TestUtil.nextInt(random, 1, 10) : Integer.MAX_VALUE, outputs, null, - willRewrite); + willRewrite, + true); for(InputOutput pair : pairs) { if (pair.output instanceof List) { Index: lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (revision 1430364) +++ lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (working copy) @@ -310,7 +310,7 @@ final boolean doRewrite = random().nextBoolean(); - Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doRewrite); + Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doRewrite, true); boolean storeOrd = random().nextBoolean(); if (VERBOSE) { @@ -453,8 +453,7 @@ this.outputs = outputs; this.doPack = doPack; - builder = new Builder(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, true, Integer.MAX_VALUE, outputs, null, doPack); - builder.setAllowArrayArcs(!noArcArrays); + builder = new Builder(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, true, Integer.MAX_VALUE, outputs, null, doPack, !noArcArrays); } protected abstract T getOutput(IntsRef input, int ord) throws IOException; @@ -1063,7 +1062,7 @@ public void testFinalOutputOnEndState() throws Exception { final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true); - final Builder builder = new Builder(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, Integer.MAX_VALUE, outputs, null, random().nextBoolean()); + final Builder builder = new Builder(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, Integer.MAX_VALUE, outputs, null, random().nextBoolean(), true); builder.add(Util.toUTF32("stat", new IntsRef()), 17L); builder.add(Util.toUTF32("station", new IntsRef()), 10L); final FST fst = builder.finish(); @@ -1078,7 +1077,7 @@ public void testInternalFinalState() throws Exception { final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true); final boolean willRewrite = random().nextBoolean(); - final Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, willRewrite); + final Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, willRewrite, true); builder.add(Util.toIntsRef(new BytesRef("stat"), new IntsRef()), outputs.getNoOutput()); builder.add(Util.toIntsRef(new BytesRef("station"), new IntsRef()), outputs.getNoOutput()); final FST fst = builder.finish(); @@ -1101,7 +1100,7 @@ final Long nothing = outputs.getNoOutput(); final Builder b = new Builder(FST.INPUT_TYPE.BYTE1, outputs); - final FST fst = new FST(FST.INPUT_TYPE.BYTE1, outputs, false, PackedInts.COMPACT); + final FST fst = new FST(FST.INPUT_TYPE.BYTE1, outputs, false, PackedInts.COMPACT, true); final Builder.UnCompiledNode rootNode = new Builder.UnCompiledNode(b, 0); Index: lucene/core/src/java/org/apache/lucene/util/fst/Builder.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/fst/Builder.java (revision 1430364) +++ lucene/core/src/java/org/apache/lucene/util/fst/Builder.java (working copy) @@ -88,7 +88,7 @@ * pruning options turned off. */ public Builder(FST.INPUT_TYPE inputType, Outputs outputs) { - this(inputType, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, false, PackedInts.COMPACT); + this(inputType, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, false, PackedInts.COMPACT, true); } /** @@ -97,9 +97,9 @@ */ public Builder(FST.INPUT_TYPE inputType, int minSuffixCount1, int minSuffixCount2, boolean doShareSuffix, boolean doShareNonSingletonNodes, int shareMaxTailLength, Outputs outputs, - FreezeTail freezeTail, boolean willPackFST) { + FreezeTail freezeTail, boolean willPackFST, boolean allowArrayArcs) { this(inputType, minSuffixCount1, minSuffixCount2, doShareSuffix, doShareNonSingletonNodes, - shareMaxTailLength, outputs, freezeTail, willPackFST, PackedInts.DEFAULT); + shareMaxTailLength, outputs, freezeTail, willPackFST, PackedInts.DEFAULT, allowArrayArcs); } /** @@ -143,10 +143,14 @@ * * @param acceptableOverheadRatio How to trade speed for space when building the FST. This option * is only relevant when doPackFST is true. @see PackedInts#getMutable(int, int, float) + * + * @param allowArrayArcs Pass false to disable the array arc optimization + * while building the FST; this will make the resulting + * FST smaller but slower to traverse. */ public Builder(FST.INPUT_TYPE inputType, int minSuffixCount1, int minSuffixCount2, boolean doShareSuffix, boolean doShareNonSingletonNodes, int shareMaxTailLength, Outputs outputs, - FreezeTail freezeTail, boolean doPackFST, float acceptableOverheadRatio) { + FreezeTail freezeTail, boolean doPackFST, float acceptableOverheadRatio, boolean allowArrayArcs) { this.minSuffixCount1 = minSuffixCount1; this.minSuffixCount2 = minSuffixCount2; this.freezeTail = freezeTail; @@ -154,7 +158,7 @@ this.shareMaxTailLength = shareMaxTailLength; this.doPackFST = doPackFST; this.acceptableOverheadRatio = acceptableOverheadRatio; - fst = new FST(inputType, outputs, doPackFST, acceptableOverheadRatio); + fst = new FST(inputType, outputs, doPackFST, acceptableOverheadRatio, allowArrayArcs); if (doShareSuffix) { dedupHash = new NodeHash(fst); } else { @@ -182,13 +186,6 @@ return dedupHash == null ? 0 : fst.nodeCount; } - /** Pass false to disable the array arc optimization - * while building the FST; this will make the resulting - * FST smaller but slower to traverse. */ - public void setAllowArrayArcs(boolean b) { - fst.setAllowArrayArcs(b); - } - private CompiledNode compileNode(UnCompiledNode nodeIn, int tailLength) throws IOException { final int node; if (dedupHash != null && (doShareNonSingletonNodes || nodeIn.numArcs <= 1) && tailLength <= shareMaxTailLength) { Index: lucene/core/src/java/org/apache/lucene/util/fst/FST.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/fst/FST.java (revision 1430364) +++ lucene/core/src/java/org/apache/lucene/util/fst/FST.java (working copy) @@ -146,6 +146,10 @@ public final Outputs outputs; + // Used for the BIT_TARGET_NEXT optimization (whereby + // instead of storing the address of the target node for + // a given arc, we mark a single bit noting that the next + // node in the byte[] is the target node): private int lastFrozenNode; private final T NO_OUTPUT; @@ -160,7 +164,7 @@ /** If arc has this label then that arc is final/accepted */ public static final int END_LABEL = -1; - private boolean allowArrayArcs = true; + private final boolean allowArrayArcs; private Arc cachedRootArcs[]; @@ -261,9 +265,10 @@ // make a new empty FST, for building; Builder invokes // this ctor - FST(INPUT_TYPE inputType, Outputs outputs, boolean willPackFST, float acceptableOverheadRatio) { + FST(INPUT_TYPE inputType, Outputs outputs, boolean willPackFST, float acceptableOverheadRatio, boolean allowArrayArcs) { this.inputType = inputType; this.outputs = outputs; + this.allowArrayArcs = allowArrayArcs; bytes = new byte[128]; NO_OUTPUT = outputs.getNoOutput(); if (willPackFST) { @@ -335,6 +340,11 @@ NO_OUTPUT = outputs.getNoOutput(); cacheRootArcs(); + + // NOTE: bogus because this is only used during + // building; we need to break out mutable FST from + // immutable + allowArrayArcs = false; } public INPUT_TYPE getInputType() { @@ -1160,10 +1170,6 @@ return arcWithOutputCount; } - public void setAllowArrayArcs(boolean v) { - allowArrayArcs = v; - } - /** * Nodes will be expanded if their depth (distance from the root node) is * <= this value and their number of arcs is >= @@ -1453,6 +1459,11 @@ this.outputs = outputs; NO_OUTPUT = outputs.getNoOutput(); writer = new DefaultBytesWriter(); + + // NOTE: bogus because this is only used during + // building; we need to break out mutable FST from + // immutable + allowArrayArcs = false; } /** Expert: creates an FST by packing this one. This Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java (revision 1430364) +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java (working copy) @@ -419,7 +419,7 @@ final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); final Builder indexBuilder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, false, Integer.MAX_VALUE, - outputs, null, false); + outputs, null, false, true); //if (DEBUG) { // System.out.println(" compile index for prefix=" + prefix); //} @@ -962,7 +962,7 @@ 0, 0, true, true, Integer.MAX_VALUE, noOutputs, - new FindBlocks(), false); + new FindBlocks(), false, true); postingsWriter.setField(fieldInfo); } Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (revision 1430364) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (working copy) @@ -113,7 +113,7 @@ this.field = field; this.doPackFST = doPackFST; this.acceptableOverheadRatio = acceptableOverheadRatio; - builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST, acceptableOverheadRatio); + builder = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST, acceptableOverheadRatio, true); } private class PostingsWriter extends PostingsConsumer { Index: lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java (revision 1430364) +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java (working copy) @@ -237,7 +237,7 @@ final Object empty = outputs.getNoOutput(); final Builder builder = new Builder( FST.INPUT_TYPE.BYTE1, 0, 0, true, true, - shareMaxTailLength, outputs, null, false); + shareMaxTailLength, outputs, null, false, true); BytesRef scratch = new BytesRef(); BytesRef entry; Index: lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java =================================================================== --- lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java (revision 1430364) +++ lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java (working copy) @@ -132,7 +132,7 @@ System.out.println(" encode..."); PositiveIntOutputs fstOutput = PositiveIntOutputs.getSingleton(true); - Builder fstBuilder = new Builder(FST.INPUT_TYPE.BYTE2, 0, 0, true, true, Integer.MAX_VALUE, fstOutput, null, true); + Builder fstBuilder = new Builder(FST.INPUT_TYPE.BYTE2, 0, 0, true, true, Integer.MAX_VALUE, fstOutput, null, true, true); IntsRef scratch = new IntsRef(); long ord = -1; // first ord will be 0 String lastValue = null;