n bytes. If n < 0 and this
+ * class supports rewinding then this method moves the stream position
+ * -n bytes backwards, otherwise it throws an
+ * {@link UnsupportedOperationException}.
+ */
+ public void skipBytes(long n) throws IOException {
+ if (n < 0) {
+ throw new UnsupportedOperationException("Cannot rewind");
+ }
+ long skipped = 0;
+ while (skipped < n) {
+ final int toSkip = (int) Math.min(n - skipped, SKIP_BUFFER.length);
+ readBytes(SKIP_BUFFER, 0, toSkip);
+ skipped += toSkip;
+ }
+ assert skipped == n;
+ }
+
/** Reads two bytes and returns a short.
* @see DataOutput#writeByte(byte)
*/
Index: lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestFastCompressionMode.java
===================================================================
--- lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestFastCompressionMode.java (révision 0)
+++ lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestFastCompressionMode.java (copie de travail)
@@ -0,0 +1,27 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TestFastCompressionMode extends AbstractTestCompressionMode {
+
+ public void setUp() throws Exception {
+ super.setUp();
+ mode = CompressionMode.FAST;
+ }
+
+}
Modification de propriétés sur lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestFastCompressionMode.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestHighCompressionMode.java
===================================================================
--- lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestHighCompressionMode.java (révision 0)
+++ lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestHighCompressionMode.java (copie de travail)
@@ -0,0 +1,27 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+public class TestHighCompressionMode extends AbstractTestCompressionMode {
+
+ public void setUp() throws Exception {
+ super.setUp();
+ mode = CompressionMode.HIGH_COMPRESSION;
+ }
+}
Modification de propriétés sur lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestHighCompressionMode.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestFastUncompressionMode.java
===================================================================
--- lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestFastUncompressionMode.java (révision 0)
+++ lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestFastUncompressionMode.java (copie de travail)
@@ -0,0 +1,28 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+public class TestFastUncompressionMode extends AbstractTestCompressionMode {
+
+ public void setUp() throws Exception {
+ super.setUp();
+ mode = CompressionMode.FAST_UNCOMPRESSION;
+ }
+
+}
Modification de propriétés sur lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestFastUncompressionMode.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
===================================================================
--- lucene/codecs/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java (révision 0)
+++ lucene/codecs/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java (copie de travail)
@@ -0,0 +1,111 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ByteArrayDataOutput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+
+public abstract class AbstractTestCompressionMode extends LuceneTestCase {
+
+ CompressionMode mode;
+
+ static byte[] randomArray() {
+ final int max = random().nextBoolean()
+ ? random().nextInt(4)
+ : random().nextInt(256);
+ final int length = random().nextBoolean()
+ ? random().nextInt(20)
+ : random().nextInt(192 * 1024);
+ final byte[] arr = new byte[length];
+ for (int i = 0; i < arr.length; ++i) {
+ arr[i] = (byte) RandomInts.randomIntBetween(random(), 0, max);
+ }
+ return arr;
+ }
+
+ byte[] compress(byte[] uncompressed) throws IOException {
+ Compressor compressor = mode.newCompressor();
+ return compress(compressor, uncompressed);
+ }
+
+ static byte[] compress(Compressor compressor, byte[] uncompressed) throws IOException {
+ byte[] compressed = new byte[uncompressed.length * 2 + 16]; // should be enough
+ ByteArrayDataOutput out = new ByteArrayDataOutput(compressed);
+ compressor.compress(uncompressed, 0, uncompressed.length, out);
+ final int compressedLen = out.getPosition();
+ return Arrays.copyOf(compressed, compressedLen);
+ }
+
+ byte[] uncompress(byte[] compressed) throws IOException {
+ Uncompressor uncompressor = mode.newUncompressor();
+ return uncompress(uncompressor, compressed);
+ }
+
+ static byte[] uncompress(Uncompressor uncompressor, byte[] compressed) throws IOException {
+ final BytesRef bytes = new BytesRef();
+ uncompressor.uncompress(new ByteArrayDataInput(compressed), bytes);
+ return Arrays.copyOfRange(bytes.bytes, bytes.offset, bytes.offset + bytes.length);
+ }
+
+ byte[] uncompress(byte[] compressed, int offset, int length) throws IOException {
+ Uncompressor uncompressor = mode.newUncompressor();
+ final BytesRef bytes = new BytesRef();
+ uncompressor.uncompress(new ByteArrayDataInput(compressed), offset, length, bytes);
+ return Arrays.copyOfRange(bytes.bytes, bytes.offset, bytes.offset + bytes.length);
+ }
+
+ public void testUncompress() throws IOException {
+ final byte[] uncompressed = randomArray();
+ final byte[] compressed = compress(uncompressed);
+ final byte[] restored = uncompress(compressed);
+ assertArrayEquals(uncompressed, restored);
+ }
+
+ public void testPartialUncompress() throws IOException {
+ final int iterations = atLeast(10);
+ for (int i = 0; i < iterations; ++i) {
+ final byte[] uncompressed = randomArray();
+ final byte[] compressed = compress(uncompressed);
+ final int offset, length;
+ if (uncompressed.length == 0) {
+ offset = length = 0;
+ } else {
+ offset = random().nextInt(uncompressed.length);
+ length = random().nextInt(uncompressed.length - offset);
+ }
+ final byte[] restored = uncompress(compressed, offset, length);
+ assertArrayEquals(Arrays.copyOfRange(uncompressed, offset, offset + length), restored);
+ }
+ }
+
+ public void testCopyCompressedData() throws IOException {
+ final byte[] uncompressed = randomArray();
+ final byte[] compressed = compress(uncompressed);
+ GrowableByteArrayDataOutput out = new GrowableByteArrayDataOutput(uncompressed.length);
+ mode.newUncompressor().copyCompressedData(new ByteArrayDataInput(compressed), out);
+ assertArrayEquals(compressed, Arrays.copyOf(out.bytes, out.length));
+ }
+
+}
Modification de propriétés sur lucene/codecs/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
===================================================================
--- lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java (révision 0)
+++ lucene/codecs/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java (copie de travail)
@@ -0,0 +1,193 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.lucene40.Lucene40Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.StorableField;
+import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+
+public class TestCompressingStoredFieldsFormat extends LuceneTestCase {
+
+ public void testWriteReadMerge() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwConf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
+ iwConf.setCodec(CompressingCodec.randomInstance(random()));
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
+ final int docCount = atLeast(200);
+ final byte[][][] data = new byte [docCount][][];
+ for (int i = 0; i < docCount; ++i) {
+ final int fieldCount = rarely()
+ ? RandomInts.randomIntBetween(random(), 1, 500)
+ : RandomInts.randomIntBetween(random(), 1, 5);
+ data[i] = new byte[fieldCount][];
+ for (int j = 0; j < fieldCount; ++j) {
+ final int length = rarely()
+ ? random().nextInt(1000)
+ : random().nextInt(10);
+ final byte[] arr = new byte[length];
+ final int max = rarely() ? 256 : 2;
+ for (int k = 0; k < length; ++k) {
+ arr[k] = (byte) random().nextInt(max);
+ }
+ data[i][j] = arr;
+ }
+ }
+
+ final FieldType type = new FieldType(StringField.TYPE_STORED);
+ type.setIndexed(false);
+ type.freeze();
+ IntField id = new IntField("id", 0, Store.YES);
+ for (int i = 0; i < data.length; ++i) {
+ Document doc = new Document();
+ doc.add(id);
+ id.setIntValue(i);
+ for (int j = 0; j < data[i].length; ++j) {
+ Field f = new Field("bytes" + j, data[i][j], type);
+ doc.add(f);
+ }
+ iw.w.addDocument(doc);
+ if (random().nextBoolean() && (i % (data.length / 10) == 0)) {
+ iw.w.close();
+ // switch codecs
+ if (iwConf.getCodec() instanceof Lucene40Codec) {
+ iwConf.setCodec(CompressingCodec.randomInstance(random()));
+ } else {
+ iwConf.setCodec(new Lucene40Codec());
+ }
+ iw = new RandomIndexWriter(random(), dir, iwConf);
+ }
+ }
+
+ for (int i = 0; i < 10; ++i) {
+ final int min = random().nextInt(data.length);
+ final int max = min + random().nextInt(20);
+ iw.deleteDocuments(NumericRangeQuery.newIntRange("id", min, max, true, false));
+ }
+
+ iw.forceMerge(2); // force merges with deletions
+
+ iw.commit();
+
+ final DirectoryReader ir = DirectoryReader.open(dir);
+ assertTrue(ir.numDocs() > 0);
+ int numDocs = 0;
+ for (int i = 0; i < ir.maxDoc(); ++i) {
+ final StoredDocument doc = ir.document(i);
+ if (doc == null) {
+ continue;
+ }
+ ++ numDocs;
+ final int docId = doc.getField("id").numericValue().intValue();
+ assertEquals(data[docId].length + 1, doc.getFields().size());
+ for (int j = 0; j < data[docId].length; ++j) {
+ final byte[] arr = data[docId][j];
+ final BytesRef arr2Ref = doc.getBinaryValue("bytes" + j);
+ final byte[] arr2 = Arrays.copyOfRange(arr2Ref.bytes, arr2Ref.offset, arr2Ref.offset + arr2Ref.length);
+ assertArrayEquals(arr, arr2);
+ }
+ }
+ assertTrue(ir.numDocs() <= numDocs);
+ ir.close();
+
+ iw.deleteAll();
+ iw.commit();
+ iw.forceMerge(1);
+ iw.close();
+ dir.close();
+ }
+
+ public void testReadSkip() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwConf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
+ iwConf.setCodec(CompressingCodec.randomInstance(random()));
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
+
+ FieldType ft = new FieldType();
+ ft.setStored(true);
+ ft.freeze();
+
+ final String string = _TestUtil.randomSimpleString(random(), 50);
+ final byte[] bytes = string.getBytes("UTF-8");
+ final long l = random().nextBoolean() ? random().nextInt(42) : random().nextLong();
+ final int i = random().nextBoolean() ? random().nextInt(42) : random().nextInt();
+ final float f = random().nextFloat();
+ final double d = random().nextDouble();
+
+ ListuncompressedLen bytes into destBytes.
+ * Please note that destBytes must be large enough to be able to hold
+ * all uncompressed data plus 8 bytes (meaning that you need to know the total
+ * uncompressed length).
+ */
+ public static void uncompress(DataInput compressed, int uncompressedLen, BytesRef destBytes) throws IOException {
+ final byte[] dest = destBytes.bytes;
+ final int destEnd = dest.length;
+ int dOff = 0;
+
+ while (dOff < uncompressedLen) {
+ // literals
+ final int token = compressed.readByte() & 0xFF;
+ int literalLen = token >>> 4;
+
+ if (literalLen != 0) {
+ if (literalLen == 0x0F) {
+ byte len;
+ while ((len = compressed.readByte()) == (byte) 0xFF) {
+ literalLen += 0xFF;
+ }
+ literalLen += len & 0xFF;
+ }
+ compressed.readBytes(dest, dOff, literalLen);
+ dOff += literalLen;
+ }
+
+ if (dOff >= uncompressedLen) {
+ break;
+ }
+
+ // matchs
+ final int matchDec = (compressed.readByte() & 0xFF) | ((compressed.readByte() & 0xFF) << 8);
+ assert matchDec > 0;
+
+ int matchLen = token & 0x0F;
+ if (matchLen == 0x0F) {
+ int len;
+ while ((len = compressed.readByte()) == (byte) 0xFF) {
+ matchLen += 0xFF;
+ }
+ matchLen += len & 0xFF;
+ }
+ matchLen += MIN_MATCH;
+
+ // copying a multiple of 8 bytes can make uncompression from 5% to 10% faster
+ final int fastLen = ((matchLen - 1) & 0xFFFFFFF8) + 8;
+ if (matchDec < matchLen || dOff + fastLen > destEnd) {
+ // overlap -> naive incremental copy
+ for (int ref = dOff - matchDec, end = dOff + matchLen; dOff < end; ++ref, ++dOff) {
+ dest[dOff] = dest[ref];
+ }
+ } else {
+ // no overlap -> arraycopy
+ System.arraycopy(dest, dOff - matchDec, dest, dOff, fastLen);
+ dOff += matchLen;
+ }
+ }
+ destBytes.offset = 0;
+ destBytes.length = dOff;
+ }
+
+ private static void encodeLen(int l, DataOutput out) throws IOException {
+ while (l >= 0xFF) {
+ out.writeByte((byte) 0xFF);
+ l -= 0xFF;
+ }
+ out.writeByte((byte) l);
+ }
+
+ private static void encodeLiterals(byte[] bytes, int token, int anchor, int literalLen, DataOutput out) throws IOException {
+ out.writeByte((byte) token);
+
+ // encode literal length
+ if (literalLen >= 0x0F) {
+ encodeLen(literalLen - 0x0F, out);
+ }
+
+ // encode literals
+ out.writeBytes(bytes, anchor, literalLen);
+ }
+
+ private static void encodeLastLiterals(byte[] bytes, int anchor, int literalLen, DataOutput out) throws IOException {
+ final int token = Math.min(literalLen, 0x0F) << 4;
+ encodeLiterals(bytes, token, anchor, literalLen, out);
+ }
+
+ private static void encodeSequence(byte[] bytes, int anchor, int matchRef, int matchOff, int matchLen, DataOutput out) throws IOException {
+ final int literalLen = matchOff - anchor;
+ assert matchLen >= 4;
+ // encode token
+ final int token = (Math.min(literalLen, 0x0F) << 4) | Math.min(matchLen - 4, 0x0F);
+ encodeLiterals(bytes, token, anchor, literalLen, out);
+
+ // encode match dec
+ final int matchDec = matchOff - matchRef;
+ assert matchDec > 0 && matchDec < 1 << 16;
+ out.writeByte((byte) matchDec);
+ out.writeByte((byte) (matchDec >>> 8));
+
+ // encode match len
+ if (matchLen >= MIN_MATCH + 0x0F) {
+ encodeLen(matchLen - 0x0F - MIN_MATCH, out);
+ }
+ }
+
+ /**
+ * Compress bytes[off:off+len] into out using
+ * 2hashLog bytes of memory. Higher values of hashLog
+ * improve the compression ratio.
+ */
+ public static void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
+
+ final int base = off;
+ final int end = off + len;
+
+ int anchor = off++;
+
+ if (len > LAST_LITERALS + MIN_MATCH) {
+
+ final int limit = end - LAST_LITERALS;
+ final int matchLimit = limit - MIN_MATCH;
+
+ final int bitsPerOffset = PackedInts.bitsRequired(len - LAST_LITERALS);
+ final int bitsPerOffsetLog = 32 - Integer.numberOfLeadingZeros(bitsPerOffset - 1);
+ final int hashLog = MEMORY_USAGE + 3 - bitsPerOffsetLog;
+ final PackedInts.Mutable hashTable = PackedInts.getMutable(1 << hashLog, bitsPerOffset, PackedInts.DEFAULT);
+
+ main:
+ while (off < limit) {
+ // find a match
+ int ref;
+ while (true) {
+ if (off >= matchLimit) {
+ break main;
+ }
+ final int v = readInt(bytes, off);
+ final int h = hash(v, hashLog);
+ ref = base + (int) hashTable.get(h);
+ assert PackedInts.bitsRequired(off - base) <= hashTable.getBitsPerValue();
+ hashTable.set(h, off - base);
+ if (off - ref < MAX_DISTANCE && readInt(bytes, ref) == v) {
+ break;
+ }
+ ++off;
+ }
+
+ // compute match length
+ final int matchLen = MIN_MATCH + commonBytes(bytes, ref + 4, off + 4, limit);
+
+ encodeSequence(bytes, anchor, ref, off, matchLen, out);
+ off += matchLen;
+ anchor = off;
+ }
+ }
+
+ // last literals
+ final int literalLen = end - anchor;
+ assert literalLen >= LAST_LITERALS || literalLen == len;
+ encodeLastLiterals(bytes, anchor, end - anchor, out);
+ }
+
+ private static class Match {
+ int start, ref, len;
+
+ void fix(int correction) {
+ start += correction;
+ ref += correction;
+ len -= correction;
+ }
+
+ int end() {
+ return start + len;
+ }
+ }
+
+ private static void copyTo(Match m1, Match m2) {
+ m2.len = m1.len;
+ m2.start = m1.start;
+ m2.ref = m1.ref;
+ }
+
+ private static class HashTable {
+ static final int MAX_ATTEMPTS = 256;
+ static final int MASK = MAX_DISTANCE - 1;
+ int nextToUpdate;
+ private final int base;
+ private final int[] hashTable;
+ private final short[] chainTable;
+
+ HashTable(int base) {
+ this.base = base;
+ nextToUpdate = base;
+ hashTable = new int[HASH_TABLE_SIZE_HC];
+ Arrays.fill(hashTable, -1);
+ chainTable = new short[MAX_DISTANCE];
+ }
+
+ private int hashPointer(byte[] bytes, int off) {
+ final int v = readInt(bytes, off);
+ final int h = hashHC(v);
+ return base + hashTable[h];
+ }
+
+ private int next(int off) {
+ return base + off - (chainTable[off & MASK] & 0xFFFF);
+ }
+
+ private void addHash(byte[] bytes, int off) {
+ final int v = readInt(bytes, off);
+ final int h = hashHC(v);
+ int delta = off - hashTable[h];
+ if (delta >= MAX_DISTANCE) {
+ delta = MAX_DISTANCE - 1;
+ }
+ chainTable[off & MASK] = (short) delta;
+ hashTable[h] = off - base;
+ }
+
+ void insert(int off, byte[] bytes) {
+ for (; nextToUpdate < off; ++nextToUpdate) {
+ addHash(bytes, nextToUpdate);
+ }
+ }
+
+ boolean insertAndFindBestMatch(byte[] buf, int off, int matchLimit, Match match) {
+ match.start = off;
+ match.len = 0;
+
+ insert(off, buf);
+
+ int ref = hashPointer(buf, off);
+ for (int i = 0; i < MAX_ATTEMPTS; ++i) {
+ if (ref < Math.max(base, off - MAX_DISTANCE + 1)) {
+ break;
+ }
+ if (buf[ref + match.len] == buf[off + match.len] && readIntEquals(buf, ref, off)) {
+ final int matchLen = MIN_MATCH + commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
+ if (matchLen > match.len) {
+ match.ref = ref;
+ match.len = matchLen;
+ }
+ }
+ ref = next(ref);
+ }
+
+ return match.len != 0;
+ }
+
+ boolean insertAndFindWiderMatch(byte[] buf, int off, int startLimit, int matchLimit, int minLen, Match match) {
+ match.len = minLen;
+
+ insert(off, buf);
+
+ final int delta = off - startLimit;
+ int ref = hashPointer(buf, off);
+ for (int i = 0; i < MAX_ATTEMPTS; ++i) {
+ if (ref < Math.max(base, off - MAX_DISTANCE + 1)) {
+ break;
+ }
+ if (buf[ref - delta + match.len] == buf[startLimit + match.len]
+ && readIntEquals(buf, ref, off)) {
+ final int matchLenForward = MIN_MATCH + commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
+ final int matchLenBackward = commonBytesBackward(buf, ref, off, base, startLimit);
+ final int matchLen = matchLenBackward + matchLenForward;
+ if (matchLen > match.len) {
+ match.len = matchLen;
+ match.ref = ref - matchLenBackward;
+ match.start = off - matchLenBackward;
+ }
+ }
+ ref = next(ref);
+ }
+
+ return match.len > minLen;
+ }
+
+ }
+
+ public static void compressHC(byte[] src, int srcOff, int srcLen, DataOutput out) throws IOException {
+
+ final int srcEnd = srcOff + srcLen;
+ final int matchLimit = srcEnd - LAST_LITERALS;
+
+ int sOff = srcOff;
+ int anchor = sOff++;
+
+ final HashTable ht = new HashTable(srcOff);
+ final Match match0 = new Match();
+ final Match match1 = new Match();
+ final Match match2 = new Match();
+ final Match match3 = new Match();
+
+ main:
+ while (sOff < matchLimit) {
+ if (!ht.insertAndFindBestMatch(src, sOff, matchLimit, match1)) {
+ ++sOff;
+ continue;
+ }
+
+ // saved, in case we would skip too much
+ copyTo(match1, match0);
+
+ search2:
+ while (true) {
+ assert match1.start >= anchor;
+ if (match1.end() >= matchLimit
+ || !ht.insertAndFindWiderMatch(src, match1.end() - 2, match1.start + 1, matchLimit, match1.len, match2)) {
+ // no better match
+ encodeSequence(src, anchor, match1.ref, match1.start, match1.len, out);
+ anchor = sOff = match1.end();
+ continue main;
+ }
+
+ if (match0.start < match1.start) {
+ if (match2.start < match1.start + match0.len) { // empirical
+ copyTo(match0, match1);
+ }
+ }
+ assert match2.start > match1.start;
+
+ if (match2.start - match1.start < 3) { // First Match too small : removed
+ copyTo(match2, match1);
+ continue search2;
+ }
+
+ search3:
+ while (true) {
+ if (match2.start - match1.start < OPTIMAL_ML) {
+ int newMatchLen = match1.len;
+ if (newMatchLen > OPTIMAL_ML) {
+ newMatchLen = OPTIMAL_ML;
+ }
+ if (match1.start + newMatchLen > match2.end() - MIN_MATCH) {
+ newMatchLen = match2.start - match1.start + match2.len - MIN_MATCH;
+ }
+ final int correction = newMatchLen - (match2.start - match1.start);
+ if (correction > 0) {
+ match2.fix(correction);
+ }
+ }
+
+ if (match2.start + match2.len >= matchLimit
+ || !ht.insertAndFindWiderMatch(src, match2.end() - 3, match2.start, matchLimit, match2.len, match3)) {
+ // no better match -> 2 sequences to encode
+ if (match2.start < match1.end()) {
+ if (match2.start - match1.start < OPTIMAL_ML) {
+ if (match1.len > OPTIMAL_ML) {
+ match1.len = OPTIMAL_ML;
+ }
+ if (match1.end() > match2.end() - MIN_MATCH) {
+ match1.len = match2.end() - match1.start - MIN_MATCH;
+ }
+ final int correction = match1.len - (match2.start - match1.start);
+ if (correction > 0) {
+ match2.fix(correction);
+ }
+ } else {
+ match1.len = match2.start - match1.start;
+ }
+ }
+ // encode seq 1
+ encodeSequence(src, anchor, match1.ref, match1.start, match1.len, out);
+ anchor = sOff = match1.end();
+ // encode seq 2
+ encodeSequence(src, anchor, match2.ref, match2.start, match2.len, out);
+ anchor = sOff = match2.end();
+ continue main;
+ }
+
+ if (match3.start < match1.end() + 3) { // Not enough space for match 2 : remove it
+ if (match3.start >= match1.end()) { // // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1
+ if (match2.start < match1.end()) {
+ final int correction = match1.end() - match2.start;
+ match2.fix(correction);
+ if (match2.len < MIN_MATCH) {
+ copyTo(match3, match2);
+ }
+ }
+
+ encodeSequence(src, anchor, match1.ref, match1.start, match1.len, out);
+ anchor = sOff = match1.end();
+
+ copyTo(match3, match1);
+ copyTo(match2, match0);
+
+ continue search2;
+ }
+
+ copyTo(match3, match2);
+ continue search3;
+ }
+
+ // OK, now we have 3 ascending matches; let's write at least the first one
+ if (match2.start < match1.end()) {
+ if (match2.start - match1.start < 0x0F) {
+ if (match1.len > OPTIMAL_ML) {
+ match1.len = OPTIMAL_ML;
+ }
+ if (match1.end() > match2.end() - MIN_MATCH) {
+ match1.len = match2.end() - match1.start - MIN_MATCH;
+ }
+ final int correction = match1.end() - match2.start;
+ match2.fix(correction);
+ } else {
+ match1.len = match2.start - match1.start;
+ }
+ }
+
+ encodeSequence(src, anchor, match1.ref, match1.start, match1.len, out);
+ anchor = sOff = match1.end();
+
+ copyTo(match2, match1);
+ copyTo(match3, match2);
+
+ continue search3;
+ }
+
+ }
+
+ }
+
+ encodeLastLiterals(src, anchor, srcEnd - anchor, out);
+ }
+
+}
Modification de propriétés sur lucene/codecs/src/java/org/apache/lucene/codecs/compressing/LZ4.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
===================================================================
--- lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (révision 0)
+++ lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (copie de travail)
@@ -0,0 +1,121 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+
+/**
+ * A {@link StoredFieldsFormat} that is very similar to
+ * {@link Lucene40StoredFieldsFormat} but compresses documents in chunks in
+ * order to improve compression ratio.
+ * + * For optimal performance, you should use a {@link MergePolicy} that returns + * segments that have the biggest byte size first. + * @lucene.experimental + */ +public class CompressingStoredFieldsFormat extends StoredFieldsFormat { + + private final CompressingStoredFieldsIndex storedFieldsIndexFormat; + private final CompressionMode compressionMode; + private final int chunkSize; + + /** + * Create a new {@link CompressingStoredFieldsFormat}. + *
+ * The compressionMode parameter allows you to choose between
+ * compression algorithms that have various compression and uncompression
+ * speeds so that you can pick the one that best fits your indexing and
+ * searching throughput.
+ *
+ * chunkSize is the minimum byte size of a chunk of documents.
+ * A value of 1 can make sense if there is redundancy across
+ * fields. In that case, both performance and compression ratio should be
+ * better than with {@link Lucene40StoredFieldsFormat} with compressed
+ * fields.
+ *
+ * Higher values of chunkSize should improve the compression
+ * atio but will require more memory at indexing time and might make document
+ * loading a little slower (depending on the size of your OS cache compared
+ * to the size of your index).
+ *
+ * The storedFieldsIndexFormat parameter allows you to choose
+ * bettwen several fields index formats that offer various trade-offs between
+ * memory usage and speed.
+ *
+ * @param compressionMode the {@link CompressionMode} to use
+ * @param chunkSize the minimum number of bytes of a single chunk of stored documents
+ * @param storedFieldsIndexFormat the format to use to load the fields index
+ * @see CompressionMode
+ * @see CompressingStoredFieldsIndex
+ */
+ public CompressingStoredFieldsFormat(CompressionMode compressionMode, int chunkSize,
+ CompressingStoredFieldsIndex storedFieldsIndexFormat) {
+ this.compressionMode = compressionMode;
+ if (chunkSize < 1) {
+ throw new IllegalArgumentException("chunkSize must be >= 1");
+ }
+ this.chunkSize = chunkSize;
+ this.storedFieldsIndexFormat = storedFieldsIndexFormat;
+ }
+
+ /**
+ * Create a new {@link CompressingStoredFieldsFormat} with an in-memory
+ * {@link CompressingStoredFieldsIndex}.
+ *
+ * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(CompressionMode, int, CompressingStoredFieldsIndex)
+ */
+ public CompressingStoredFieldsFormat(CompressionMode compressionMode, int chunkSize) {
+ this (compressionMode, chunkSize, chunkSize == 1
+ ? CompressingStoredFieldsIndex.MEMORY_DOC
+ : CompressingStoredFieldsIndex.MEMORY_CHUNK);
+ }
+
+ /**
+ * Create a new {@link CompressingStoredFieldsFormat} with
+ * {@link CompressionMode#FAST} compression and chunks of 16 KB.
+ *
+ * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(CompressionMode, int)
+ */
+ public CompressingStoredFieldsFormat() {
+ this(CompressionMode.FAST, 1 << 14);
+ }
+
+ @Override
+ public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si,
+ FieldInfos fn, IOContext context) throws IOException {
+ return new CompressingStoredFieldsReader(directory, si, fn, context);
+ }
+
+ @Override
+ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si,
+ IOContext context) throws IOException {
+ return new CompressingStoredFieldsWriter(directory, si, context,
+ compressionMode, chunkSize, storedFieldsIndexFormat);
+ }
+
+}
Modification de propriétés sur lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/java/org/apache/lucene/codecs/compressing/package.html
===================================================================
--- lucene/codecs/src/java/org/apache/lucene/codecs/compressing/package.html (révision 0)
+++ lucene/codecs/src/java/org/apache/lucene/codecs/compressing/package.html (copie de travail)
@@ -0,0 +1,25 @@
+
+
+
+
+ * These formats allow different memory/speed trade-offs to locate documents + * into the fields data file (.fdt). + * @lucene.experimental + */ +public enum CompressingStoredFieldsIndex { + + /** + * This format stores the document index on disk using 64-bits pointers to + * the start offsets of chunks in the fields data file. + *
+ * This format has no memory overhead and requires at most 1 disk seek to + * locate a document in the fields data file. Use this format in + * memory-constrained environments. + */ + DISK_DOC(0) { + @Override + Writer newWriter(IndexOutput out) { + return new DiskDocFieldsIndexWriter(out); + } + @Override + Reader newReader(IndexInput in, SegmentInfo si) throws IOException { + return new DiskDocFieldsIndexReader(in, si); + } + }, + + /** + * For every document in the segment, this format stores the offset of the + * compressed chunk that contains it in the fields data file. + *
+ * This fields index format requires at most 8 × numDocs bytes
+ * of memory. Locating a document in the fields data file requires no disk
+ * seek. Use this format when blocks are very likely to contain few
+ * documents (in particular when chunkSize = 1).
+ */
+ MEMORY_DOC(1) {
+ @Override
+ Writer newWriter(IndexOutput out) throws IOException {
+ return new MemoryDocFieldsIndexWriter(out);
+ }
+ @Override
+ Reader newReader(IndexInput in, SegmentInfo si) throws IOException {
+ return new MemoryDocFieldsIndexReader(in, si);
+ }
+ },
+
+ /**
+ * For every chunk of compressed documents, this format stores the first doc
+ * ID of the chunk as well as the start offset of the chunk.
+ *
+ * This fields index format require at most
+ * 12 × numChunks bytes of memory. Locating a document in the
+ * fields data file requires no disk seek. Use this format when chunks are
+ * likely to contain several documents.
+ */
+ MEMORY_CHUNK(2) {
+ @Override
+ Writer newWriter(IndexOutput out) throws IOException {
+ return new MemoryChunkFieldsIndexWriter(out);
+ }
+ @Override
+ Reader newReader(IndexInput in, SegmentInfo si) throws IOException {
+ return new MemoryChunkFieldsIndexReader(in, si);
+ }
+ };
+
+ /**
+ * Retrieve a {@link CompressingStoredFieldsIndex} according to its
+ * ID.
+ */
+ public static CompressingStoredFieldsIndex byId(int id) {
+ for (CompressingStoredFieldsIndex idx : CompressingStoredFieldsIndex.values()) {
+ if (idx.getId() == id) {
+ return idx;
+ }
+ }
+ throw new IllegalArgumentException("Unknown id: " + id);
+ }
+
+ private final int id;
+
+ private CompressingStoredFieldsIndex(int id) {
+ this.id = id;
+ }
+
+ /**
+ * Returns an ID for this compression mode. Should be unique across
+ * {@link CompressionMode}s as it is used for serialization and
+ * unserialization.
+ */
+ public final int getId() {
+ return id;
+ }
+
+ abstract Writer newWriter(IndexOutput out) throws IOException;
+
+ abstract Reader newReader(IndexInput in, SegmentInfo si) throws IOException;
+
+ static abstract class Writer implements Closeable {
+
+ protected final IndexOutput fieldsIndexOut;
+
+ Writer(IndexOutput indexOutput) {
+ this.fieldsIndexOut = indexOutput;
+ }
+
+ /** Write the index file for a chunk of numDocs docs starting
+ * at offset startPointer. */
+ abstract void writeIndex(int numDocs, long startPointer) throws IOException;
+
+ /** Finish writing an index file of numDocs documents. */
+ void finish(int numDocs) throws IOException {}
+
+ @Override
+ public void close() throws IOException {
+ fieldsIndexOut.close();
+ }
+
+ }
+
+ private static class DiskDocFieldsIndexWriter extends Writer {
+
+ private final long startOffset;
+
+ DiskDocFieldsIndexWriter(IndexOutput fieldsIndexOut) {
+ super(fieldsIndexOut);
+ startOffset = fieldsIndexOut.getFilePointer();
+ }
+
+ @Override
+ void writeIndex(int numDocs, long startPointer) throws IOException {
+ for (int i = 0; i < numDocs; ++i) {
+ fieldsIndexOut.writeLong(startPointer);
+ }
+ }
+
+ @Override
+ void finish(int numDocs) throws IOException {
+ if (startOffset + ((long) numDocs) * 8 != fieldsIndexOut.getFilePointer()) {
+ // see Lucene40StoredFieldsWriter#finish
+ throw new RuntimeException((fieldsIndexOut.getFilePointer() - startOffset)/8 + " fdx size mismatch: docCount is " + numDocs + " but fdx file size is " + fieldsIndexOut.getFilePointer() + " file=" + fieldsIndexOut.toString() + "; now aborting this merge to prevent index corruption");
+ }
+ }
+
+ }
+
+ private static class MemoryDocFieldsIndexWriter extends Writer {
+
+ protected final long indexStartPointer;
+ protected long lastStartPointer;
+
+ MemoryDocFieldsIndexWriter(IndexOutput indexOutput) throws IOException {
+ super(indexOutput);
+ lastStartPointer = 0;
+ indexStartPointer = fieldsIndexOut.getFilePointer();
+ fieldsIndexOut.writeByte((byte) -1); // the number of bits per start pointer
+ }
+
+ @Override
+ void writeIndex(int numDocs, long startPointer) throws IOException {
+ assert startPointer > lastStartPointer;
+ fieldsIndexOut.writeVInt(numDocs);
+ fieldsIndexOut.writeVLong(startPointer - lastStartPointer);
+ lastStartPointer = startPointer;
+ }
+
+ @Override
+ void finish(int numDocs) throws IOException {
+ fieldsIndexOut.writeVInt(0); // stop marker
+ fieldsIndexOut.seek(indexStartPointer);
+ fieldsIndexOut.writeByte((byte) PackedInts.bitsRequired(lastStartPointer));
+ }
+
+ }
+
+ private static class MemoryChunkFieldsIndexWriter extends MemoryDocFieldsIndexWriter {
+
+ private int numChunks = 0;
+
+ MemoryChunkFieldsIndexWriter(IndexOutput indexOutput) throws IOException {
+ super(indexOutput);
+ fieldsIndexOut.writeInt(-1);
+ }
+
+ @Override
+ void writeIndex(int numDocs, long startPointer) throws IOException {
+ super.writeIndex(numDocs, startPointer);
+ ++numChunks;
+ }
+
+ @Override
+ void finish(int numDocs) throws IOException {
+ fieldsIndexOut.writeVInt(0); // stop marker
+ fieldsIndexOut.seek(indexStartPointer);
+ fieldsIndexOut.writeByte((byte) PackedInts.bitsRequired(lastStartPointer));
+ fieldsIndexOut.writeInt(numChunks);
+ }
+
+ }
+
+ static abstract class Reader implements Cloneable, Closeable {
+
+ protected final IndexInput fieldsIndexIn;
+
+ Reader(IndexInput fieldsIndexIn) {
+ this.fieldsIndexIn = fieldsIndexIn;
+ }
+
+ /** Get the start pointer of the compressed block that contains docID */
+ abstract long getStartPointer(int docID) throws IOException;
+
+ public void close() throws IOException {
+ if (fieldsIndexIn != null) {
+ fieldsIndexIn.close();
+ }
+ }
+
+ public abstract Reader clone();
+ }
+
+ private static class DiskDocFieldsIndexReader extends Reader {
+
+ private final long startPointer;
+
+ DiskDocFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws CorruptIndexException {
+ this(fieldsIndexIn, fieldsIndexIn.getFilePointer());
+ final long indexSize = fieldsIndexIn.length() - fieldsIndexIn.getFilePointer();
+ final int numDocs = (int) (indexSize >> 3);
+ // Verify two sources of "maxDoc" agree:
+ if (numDocs != si.getDocCount()) {
+ throw new CorruptIndexException("doc counts differ for segment " + si + ": fieldsReader shows " + numDocs + " but segmentInfo shows " + si.getDocCount());
+ }
+ }
+
+ private DiskDocFieldsIndexReader(IndexInput fieldsIndexIn, long startPointer) {
+ super(fieldsIndexIn);
+ this.startPointer = startPointer;
+ }
+
+ @Override
+ long getStartPointer(int docID) throws IOException {
+ fieldsIndexIn.seek(startPointer + docID * 8L);
+ return fieldsIndexIn.readLong();
+ }
+
+ @Override
+ public Reader clone() {
+ return new DiskDocFieldsIndexReader(fieldsIndexIn.clone(), startPointer);
+ }
+
+ }
+
+ private static class MemoryDocFieldsIndexReader extends Reader {
+
+ private final PackedInts.Reader startPointers;
+
+ MemoryDocFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {
+ super(fieldsIndexIn);
+ final int bitsPerStartPointer = fieldsIndexIn.readByte() & 0xFF;
+ if (bitsPerStartPointer > 64) {
+ throw new IOException("Corrupted");
+ }
+ PackedInts.Mutable startPointers = PackedInts.getMutable(si.getDocCount(), bitsPerStartPointer, PackedInts.COMPACT);
+ long startPointer = 0;
+ int i = 0;
+ while (true) {
+ final int chunkDocs = fieldsIndexIn.readVInt();
+ if (chunkDocs == 0) {
+ break;
+ }
+ if (i + chunkDocs > si.getDocCount()) {
+ throw new IOException("Corrupted");
+ }
+ startPointer += fieldsIndexIn.readVLong();
+ for (int j = 0; j < chunkDocs; ++j) {
+ startPointers.set(i++, startPointer);
+ }
+ }
+ if (i != si.getDocCount()) {
+ throw new IOException("Corrupted: expected " + si.getDocCount() + " documents for segment " + si + ", but got " + i);
+ }
+ this.startPointers = startPointers;
+ }
+
+ private MemoryDocFieldsIndexReader(PackedInts.Reader startPointers) {
+ super(null);
+ this.startPointers = startPointers;
+ }
+
+ @Override
+ long getStartPointer(int docID) throws IOException {
+ return startPointers.get(docID);
+ }
+
+ @Override
+ public Reader clone() {
+ if (fieldsIndexIn == null) {
+ return this;
+ } else {
+ return new MemoryDocFieldsIndexReader(startPointers);
+ }
+ }
+
+ }
+
+ private static class MemoryChunkFieldsIndexReader extends Reader {
+
+ private final PackedInts.Reader docBases;
+ private final PackedInts.Reader startPointers;
+
+ MemoryChunkFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {
+ super(fieldsIndexIn);
+ final int bitsPerStartPointer = fieldsIndexIn.readByte() & 0xFF;
+ if (bitsPerStartPointer > 64) {
+ throw new IOException("Corrupted");
+ }
+ final int numChunks = fieldsIndexIn.readInt();
+ if (numChunks < 0 || numChunks > si.getDocCount() || (numChunks == 0 && si.getDocCount() > 0)) {
+ throw new IOException("Corrupted, more chunks than docs: " + numChunks + " > " + si.getDocCount());
+ }
+ PackedInts.Mutable docBases = PackedInts.getMutable(numChunks, PackedInts.bitsRequired(Math.max(0, si.getDocCount() - 1)), PackedInts.COMPACT);
+ PackedInts.Mutable startPointers = PackedInts.getMutable(numChunks, bitsPerStartPointer, PackedInts.COMPACT);
+ int docBase = 0;
+ long startPointer = 0;
+ int i = 0;
+ while (true) {
+ final int chunkDocs = fieldsIndexIn.readVInt();
+ if (chunkDocs == 0) {
+ break;
+ }
+ if (i > numChunks) {
+ throw new IOException("Corrupted");
+ }
+ startPointer += fieldsIndexIn.readVLong();
+ docBases.set(i, docBase);
+ startPointers.set(i, startPointer);
+ docBase += chunkDocs;
+ ++i;
+ }
+ if (docBase != si.getDocCount()) {
+ throw new IOException("Corrupted: expected " + si.getDocCount() + " documents for segment " + si + ", but got " + docBase);
+ }
+ this.docBases = docBases;
+ this.startPointers = startPointers;
+ }
+
+ private MemoryChunkFieldsIndexReader(PackedInts.Reader docBases, PackedInts.Reader startPointers) {
+ super(null);
+ this.docBases = docBases;
+ this.startPointers = startPointers;
+ }
+
+ @Override
+ long getStartPointer(int docID) {
+ assert docBases.size() > 0;
+ int lo = 0, hi = docBases.size() - 1;
+ while (lo <= hi) {
+ final int mid = (lo + hi) >>> 1;
+ final long midValue = docBases.get(mid);
+ if (midValue == docID) {
+ return startPointers.get(mid);
+ } else if (midValue < docID) {
+ lo = mid + 1;
+ } else {
+ hi = mid - 1;
+ }
+ }
+ return startPointers.get(hi);
+ }
+
+ @Override
+ public Reader clone() {
+ if (fieldsIndexIn == null) {
+ return this;
+ } else {
+ return new MemoryChunkFieldsIndexReader(docBases, startPointers);
+ }
+ }
+
+ }
+
+}
Modification de propriétés sur lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndex.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/java/org/apache/lucene/codecs/compressing/Compressor.java
===================================================================
--- lucene/codecs/src/java/org/apache/lucene/codecs/compressing/Compressor.java (révision 0)
+++ lucene/codecs/src/java/org/apache/lucene/codecs/compressing/Compressor.java (copie de travail)
@@ -0,0 +1,36 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.store.DataOutput;
+
+/**
+ * A data compressor.
+ */
+abstract class Compressor {
+
+ /**
+ * Compress bytes into out. It it the responsibility of the
+ * compressor to add all necessary information so that a {@link Uncompressor}
+ * will know when to stop uncompressing bytes from the stream.
+ */
+ public abstract void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException;
+
+}
Modification de propriétés sur lucene/codecs/src/java/org/apache/lucene/codecs/compressing/Compressor.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
===================================================================
--- lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (révision 0)
+++ lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (copie de travail)
@@ -0,0 +1,390 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsWriter.FIELDS_EXTENSION;
+import static org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION;
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.ChunkIterator;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.StorableField;
+import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.PackedInts;
+
+final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
+
+ static final int STRING = 0x00;
+ static final int BYTE_ARR = 0x01;
+ static final int NUMERIC_INT = 0x02;
+ static final int NUMERIC_FLOAT = 0x03;
+ static final int NUMERIC_LONG = 0x04;
+ static final int NUMERIC_DOUBLE = 0x05;
+
+ static final int TYPE_BITS = PackedInts.bitsRequired(NUMERIC_DOUBLE);
+ static final int TYPE_MASK = (int) PackedInts.maxValue(TYPE_BITS);
+
+ static final String CODEC_NAME_IDX = "CompressingStoredFieldsIndex";
+ static final String CODEC_NAME_DAT = "CompressingStoredFieldsData";
+ static final int VERSION_START = 0;
+ static final int VERSION_CURRENT = VERSION_START;
+ static final long HEADER_LENGTH_IDX = CodecUtil.headerLength(CODEC_NAME_IDX);
+ static final long HEADER_LENGTH_DAT = CodecUtil.headerLength(CODEC_NAME_DAT);
+
+ private final Directory directory;
+ private final String segment;
+ private CompressingStoredFieldsIndex.Writer indexWriter;
+ private IndexOutput fieldsStream;
+
+ private final CompressionMode compressionMode;
+ private final Compressor compressor;
+ private final int chunkSize;
+
+ private final GrowableByteArrayDataOutput bufferedDocs;
+ private int[] endOffsets; // end offsets in bufferedDocs
+ private int docBase; // doc ID at the beginning of the chunk
+ private int numBufferedDocs; // docBase + numBufferedDocs == current doc ID
+
+ public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si,
+ IOContext context, CompressionMode compressionMode, int chunkSize, CompressingStoredFieldsIndex storedFieldsIndex) throws IOException {
+ assert directory != null;
+ this.directory = directory;
+ this.segment = si.name;
+ this.compressionMode = compressionMode;
+ this.compressor = compressionMode.newCompressor();
+ this.chunkSize = chunkSize;
+ this.docBase = 0;
+ this.bufferedDocs = new GrowableByteArrayDataOutput(chunkSize);
+ this.endOffsets = new int[16];
+ this.numBufferedDocs = 0;
+
+ boolean success = false;
+ IndexOutput indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION), context);
+ try {
+ fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context);
+
+ CodecUtil.writeHeader(indexStream, CODEC_NAME_IDX, VERSION_CURRENT);
+ CodecUtil.writeHeader(fieldsStream, CODEC_NAME_DAT, VERSION_CURRENT);
+ assert HEADER_LENGTH_IDX == indexStream.getFilePointer();
+ assert HEADER_LENGTH_DAT == fieldsStream.getFilePointer();
+
+ indexStream.writeVInt(storedFieldsIndex.getId());
+ indexWriter = storedFieldsIndex.newWriter(indexStream);
+ indexStream = null;
+
+ fieldsStream.writeVInt(PackedInts.VERSION_CURRENT);
+ fieldsStream.writeVInt(compressionMode.getId());
+
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(indexStream);
+ abort();
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ IOUtils.close(fieldsStream, indexWriter);
+ } finally {
+ fieldsStream = null;
+ indexWriter = null;
+ }
+ }
+
+ private void endWithPreviousDocument() throws IOException {
+ if (numBufferedDocs > 0) {
+ assert bufferedDocs.length > 0;
+ if (numBufferedDocs == endOffsets.length) {
+ endOffsets = ArrayUtil.grow(endOffsets);
+ }
+ endOffsets[numBufferedDocs - 1] = bufferedDocs.length;
+ }
+ }
+
+ private void addRawDocument(byte[] buf, int off, int len) throws IOException {
+ endWithPreviousDocument();
+ if (bufferedDocs.length >= chunkSize) {
+ flush();
+ }
+ bufferedDocs.writeBytes(buf, off, len);
+ ++numBufferedDocs;
+ }
+
+ @Override
+ public void startDocument(int numStoredFields) throws IOException {
+ endWithPreviousDocument();
+ if (bufferedDocs.length >= chunkSize) {
+ flush();
+ }
+ bufferedDocs.writeVInt(numStoredFields);
+ ++numBufferedDocs;
+ }
+
+ private void writeHeader(int docBase, int numBufferedDocs, int[] lengths) throws IOException {
+ // save docBase and numBufferedDocs
+ fieldsStream.writeVInt(docBase);
+ fieldsStream.writeVInt(numBufferedDocs);
+
+ // save lengths
+ final int bitsRequired = bitsRequired(lengths, numBufferedDocs);
+ assert bitsRequired <= 31;
+ fieldsStream.writeVInt(bitsRequired);
+
+ final PackedInts.Writer writer = PackedInts.getWriterNoHeader(fieldsStream, PackedInts.Format.PACKED, numBufferedDocs, bitsRequired, 1);
+ for (int i = 0; i < numBufferedDocs; ++i) {
+ assert lengths[i] > 0;
+ writer.add(lengths[i]);
+ }
+ assert writer.ord() + 1 == numBufferedDocs;
+ writer.finish();
+ }
+
+ private void flush() throws IOException {
+ indexWriter.writeIndex(numBufferedDocs, fieldsStream.getFilePointer());
+
+ // transform end offsets into lengths
+ final int[] lengths = endOffsets;
+ for (int i = numBufferedDocs - 1; i > 0; --i) {
+ lengths[i] = endOffsets[i] - endOffsets[i - 1];
+ assert lengths[i] > 0;
+ }
+ writeHeader(docBase, numBufferedDocs, lengths);
+
+ // compress stored fields to fieldsStream
+ compressor.compress(bufferedDocs.bytes, 0, bufferedDocs.length, fieldsStream);
+
+ // reset
+ docBase += numBufferedDocs;
+ numBufferedDocs = 0;
+ bufferedDocs.length = 0;
+ }
+
+ private static int bitsRequired(int[] data, int length) {
+ int or = data[0];
+ for (int i = 1; i < length; ++i) {
+ or |= data[i];
+ }
+ return PackedInts.bitsRequired(or);
+ }
+
+ @Override
+ public void writeField(FieldInfo info, StorableField field)
+ throws IOException {
+ int bits = 0;
+ final BytesRef bytes;
+ final String string;
+
+ Number number = field.numericValue();
+ if (number != null) {
+ if (number instanceof Byte || number instanceof Short || number instanceof Integer) {
+ bits = NUMERIC_INT;
+ } else if (number instanceof Long) {
+ bits = NUMERIC_LONG;
+ } else if (number instanceof Float) {
+ bits = NUMERIC_FLOAT;
+ } else if (number instanceof Double) {
+ bits = NUMERIC_DOUBLE;
+ } else {
+ throw new IllegalArgumentException("cannot store numeric type " + number.getClass());
+ }
+ string = null;
+ bytes = null;
+ } else {
+ bytes = field.binaryValue();
+ if (bytes != null) {
+ bits = BYTE_ARR;
+ string = null;
+ } else {
+ bits = STRING;
+ string = field.stringValue();
+ if (string == null) {
+ throw new IllegalArgumentException("field " + field.name() + " is stored but does not have binaryValue, stringValue nor numericValue");
+ }
+ }
+ }
+
+ final long infoAndBits = (((long) info.number) << TYPE_BITS) | bits;
+ bufferedDocs.writeVLong(infoAndBits);
+
+ if (bytes != null) {
+ bufferedDocs.writeVInt(bytes.length);
+ bufferedDocs.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ } else if (string != null) {
+ bufferedDocs.writeString(field.stringValue());
+ } else {
+ if (number instanceof Byte || number instanceof Short || number instanceof Integer) {
+ bufferedDocs.writeInt(number.intValue());
+ } else if (number instanceof Long) {
+ bufferedDocs.writeLong(number.longValue());
+ } else if (number instanceof Float) {
+ bufferedDocs.writeInt(Float.floatToIntBits(number.floatValue()));
+ } else if (number instanceof Double) {
+ bufferedDocs.writeLong(Double.doubleToLongBits(number.doubleValue()));
+ } else {
+ throw new AssertionError("Cannot get here");
+ }
+ }
+ }
+
+ @Override
+ public void abort() {
+ IOUtils.closeWhileHandlingException(this);
+ IOUtils.deleteFilesIgnoringExceptions(directory,
+ IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION),
+ IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION));
+ }
+
+ @Override
+ public void finish(FieldInfos fis, int numDocs) throws IOException {
+ endWithPreviousDocument();
+ if (numBufferedDocs > 0) {
+ flush();
+ }
+ if (docBase != numDocs) {
+ throw new RuntimeException("Wrote " + docBase + " docs, finish called with numDocs=" + numDocs);
+ }
+ indexWriter.finish(numDocs);
+ assert bufferedDocs.length == 0;
+ }
+
+ @Override
+ public int merge(MergeState mergeState) throws IOException {
+ int docCount = 0;
+ int idx = 0;
+
+ for (AtomicReader reader : mergeState.readers) {
+ final SegmentReader matchingSegmentReader = mergeState.matchingSegmentReaders[idx++];
+ CompressingStoredFieldsReader matchingFieldsReader = null;
+ if (matchingSegmentReader != null) {
+ final StoredFieldsReader fieldsReader = matchingSegmentReader.getFieldsReader();
+ // we can only bulk-copy if the matching reader is also a CompressingStoredFieldsReader
+ if (fieldsReader != null && fieldsReader instanceof CompressingStoredFieldsReader) {
+ matchingFieldsReader = (CompressingStoredFieldsReader) fieldsReader;
+ }
+ }
+
+ final int maxDoc = reader.maxDoc();
+ final Bits liveDocs = reader.getLiveDocs();
+
+ if (matchingFieldsReader == null) {
+ // naive merge...
+ for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) {
+ StoredDocument doc = reader.document(i);
+ addDocument(doc, mergeState.fieldInfos);
+ ++docCount;
+ mergeState.checkAbort.work(300);
+ }
+ } else {
+ int docID = nextLiveDoc(0, liveDocs, maxDoc);
+ if (docID < maxDoc) {
+ // not all docs were deleted
+ final ChunkIterator it = matchingFieldsReader.chunkIterator(docID);
+ int[] startOffsets = new int[0];
+ do {
+ // go to the next chunk that contains docID
+ it.next(docID);
+ // transform lengths into offsets
+ if (startOffsets.length < it.chunkDocs) {
+ startOffsets = new int[ArrayUtil.oversize(it.chunkDocs, 4)];
+ }
+ for (int i = 1; i < it.chunkDocs; ++i) {
+ startOffsets[i] = startOffsets[i - 1] + it.lengths[i - 1];
+ }
+
+ if (compressionMode == matchingFieldsReader.getCompressionMode() // same compression mode
+ && (numBufferedDocs == 0 || bufferedDocs.length >= chunkSize) // starting a new chunk
+ && startOffsets[it.chunkDocs - 1] < chunkSize // chunk is small enough
+ && startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] >= chunkSize // chunk is large enough
+ && nextDeletedDoc(it.docBase, liveDocs, it.docBase + it.chunkDocs) == it.docBase + it.chunkDocs) { // no deletion in the chunk
+ assert docID == it.docBase;
+
+ // no need to uncompress, just copy data
+ endWithPreviousDocument();
+ if (bufferedDocs.length >= chunkSize) {
+ flush();
+ }
+ indexWriter.writeIndex(it.chunkDocs, fieldsStream.getFilePointer());
+ writeHeader(this.docBase, it.chunkDocs, it.lengths);
+ it.copyCompressedData(fieldsStream);
+ this.docBase += it.chunkDocs;
+ docID = nextLiveDoc(it.docBase + it.chunkDocs, liveDocs, maxDoc);
+ docCount += it.chunkDocs;
+ mergeState.checkAbort.work(300 * it.chunkDocs);
+ } else {
+ // uncompress
+ it.uncompress();
+ if (startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] != it.bytes.length) {
+ throw new IOException("Corrupted: expected chunk size=" + startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] + ", got " + it.bytes.length);
+ }
+ // copy non-deleted docs
+ for (; docID < it.docBase + it.chunkDocs; docID = nextLiveDoc(docID + 1, liveDocs, maxDoc)) {
+ final int diff = docID - it.docBase;
+ addRawDocument(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
+ ++docCount;
+ mergeState.checkAbort.work(300);
+ }
+ }
+ } while (docID < maxDoc);
+ }
+ }
+ }
+ finish(mergeState.fieldInfos, docCount);
+ return docCount;
+ }
+
+ private static int nextLiveDoc(int doc, Bits liveDocs, int maxDoc) {
+ if (liveDocs == null) {
+ return doc;
+ }
+ while (doc < maxDoc && !liveDocs.get(doc)) {
+ ++doc;
+ }
+ return doc;
+ }
+
+ private static int nextDeletedDoc(int doc, Bits liveDocs, int maxDoc) {
+ if (liveDocs == null) {
+ return maxDoc;
+ }
+ while (doc < maxDoc && liveDocs.get(doc)) {
+ ++doc;
+ }
+ return doc;
+ }
+
+}
Modification de propriétés sur lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/java/org/apache/lucene/codecs/compressing/Uncompressor.java
===================================================================
--- lucene/codecs/src/java/org/apache/lucene/codecs/compressing/Uncompressor.java (révision 0)
+++ lucene/codecs/src/java/org/apache/lucene/codecs/compressing/Uncompressor.java (copie de travail)
@@ -0,0 +1,56 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * An uncompressor.
+ */
+abstract class Uncompressor implements Cloneable {
+
+ /**
+ * Uncompress bytes. This method is free to resize bytes in case
+ * it is too small to hold all the uncompressed data.
+ */
+ public abstract void uncompress(DataInput in, BytesRef bytes) throws IOException;
+
+ /**
+ * Method to use if you are only interested into length
+ * uncompressed bytes starting at offset offset. Some compression
+ * codecs might have optimizations for this special case.
+ */
+ public void uncompress(DataInput in, int offset, int length, BytesRef bytes) throws IOException {
+ uncompress(in, bytes);
+ if (bytes.length < offset + length) {
+ throw new IndexOutOfBoundsException((offset + length) + " > " + bytes.length);
+ }
+ bytes.offset += offset;
+ bytes.length = length;
+ }
+
+ public abstract void copyCompressedData(DataInput in, DataOutput out) throws IOException;
+
+ @Override
+ public abstract Uncompressor clone();
+
+}
Modification de propriétés sur lucene/codecs/src/java/org/apache/lucene/codecs/compressing/Uncompressor.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/codecs/src/java/org/apache/lucene/codecs/compressing/GrowableByteArrayDataOutput.java
===================================================================
--- lucene/codecs/src/java/org/apache/lucene/codecs/compressing/GrowableByteArrayDataOutput.java (révision 0)
+++ lucene/codecs/src/java/org/apache/lucene/codecs/compressing/GrowableByteArrayDataOutput.java (copie de travail)
@@ -0,0 +1,56 @@
+package org.apache.lucene.codecs.compressing;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.ArrayUtil;
+
+/**
+ * A {@link DataOutput} that can be used to build a byte[].
+ */
+final class GrowableByteArrayDataOutput extends DataOutput {
+
+ byte[] bytes;
+ int length;
+
+ GrowableByteArrayDataOutput(int cp) {
+ this.bytes = new byte[ArrayUtil.oversize(cp, 1)];
+ this.length = 0;
+ }
+
+ @Override
+ public void writeByte(byte b) throws IOException {
+ if (length >= bytes.length) {
+ bytes = ArrayUtil.grow(bytes);
+ }
+ bytes[length++] = b;
+ }
+
+ @Override
+ public void writeBytes(byte[] b, int off, int len) throws IOException {
+ final int newLength = length + len;
+ if (newLength > bytes.length) {
+ bytes = ArrayUtil.grow(bytes, newLength);
+ }
+ System.arraycopy(b, off, bytes, length, len);
+ length = newLength;
+ }
+
+}
Modification de propriétés sur lucene/codecs/src/java/org/apache/lucene/codecs/compressing/GrowableByteArrayDataOutput.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property