commit 01988012fa9c114130479ac9640957e825486769 Author: Daniel Lemire Date: Mon Sep 28 10:10:36 2015 -0400 Replacing Concise with Roaring throughout. diff --git a/cube/pom.xml b/cube/pom.xml index 2e8ffb0..e5c1f91 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -89,9 +89,13 @@ compress-lzf + org.roaringbitmap + RoaringBitmap + + diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml index 5d1cf6d..0eaaf1e 100644 --- a/invertedindex/pom.xml +++ b/invertedindex/pom.xml @@ -87,9 +87,13 @@ compress-lzf + org.roaringbitmap + RoaringBitmap + + @@ -158,4 +162,4 @@ - \ No newline at end of file + diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java index 2345261..b7f4ea5 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java @@ -18,8 +18,7 @@ package org.apache.kylin.invertedindex.index; -import java.nio.ByteBuffer; -import java.nio.IntBuffer; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -28,7 +27,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.dict.Dictionary; -import it.uniroma3.mat.extendedset.intset.ConciseSet; +import org.roaringbitmap.RoaringBitmap; /** * @author yangli9 @@ -38,7 +37,7 @@ public class BitMapContainer implements ColumnValueContainer { int valueLen; int nValues; int size; - ConciseSet[] sets; + RoaringBitmap[] sets; boolean closedForChange; transient byte[] temp; @@ -76,7 +75,7 @@ public class BitMapContainer implements ColumnValueContainer { } @Override - public ConciseSet getBitMap(Integer startId, Integer endId) { + public RoaringBitmap getBitMap(Integer startId, Integer endId) { if (startId == null && endId == null) { return sets[this.nValues]; } @@ -90,15 +89,10 @@ public class BitMapContainer implements ColumnValueContainer { end = endId; } - ConciseSet ret = new ConciseSet(); - for (int i = start; i <= end; ++i) { - ConciseSet temp = getBitMap(i); - ret.addAll(temp); - } - return ret; + return RoaringBitmap.or(Arrays.copyOfRange(sets, start, end+1)); } - private ConciseSet getBitMap(int valueId) { + private RoaringBitmap getBitMap(int valueId) { if (valueId >= 0 && valueId <= getMaxValueId()) return sets[valueId]; else @@ -125,9 +119,9 @@ public class BitMapContainer implements ColumnValueContainer { throw new IllegalStateException(); } if (sets == null) { - sets = new ConciseSet[nValues + 1]; + sets = new RoaringBitmap[nValues + 1]; for (int i = 0; i <= nValues; i++) { - sets[i] = new ConciseSet(); + sets[i] = new RoaringBitmap(); } } } @@ -159,34 +153,92 @@ public class BitMapContainer implements ColumnValueContainer { public void fromBytes(List bytes) { assert nValues + 1 == bytes.size(); - sets = new ConciseSet[nValues + 1]; + sets = new RoaringBitmap[nValues + 1]; size = 0; for (int i = 0; i <= nValues; i++) { sets[i] = bytesToSet(bytes.get(i)); - size += sets[i].size(); + size += sets[i].getCardinality(); } closedForChange = true; } - private ImmutableBytesWritable setToBytes(ConciseSet set) { - byte[] array; - if (set.isEmpty()) // ConciseSet.toByteBuffer() throws exception when - // set is empty - array = BytesUtil.EMPTY_BYTE_ARRAY; - else - array = set.toByteBuffer().array(); + private ImmutableBytesWritable setToBytes(RoaringBitmap set) { + // Serializing a bitmap to a byte array can be expected to be expensive, this should not be commonly done. + // If the purpose is to save the data to disk or to a network, then a direct serialization would be + // far more efficient. If the purpose is to enforce immutability, it is an expensive way to do it. + set.runOptimize(); //to improve compression + final byte[] array = new byte[set.serializedSizeInBytes()]; + try { + set.serialize(new java.io.DataOutputStream(new java.io.OutputStream() { + int c = 0; + + @Override + public void close() { + } + + @Override + public void flush() { + } + + @Override + public void write(int b) { + array[c++] = (byte)b; + } + + @Override + public void write(byte[] b) { + write(b, 0, b.length); + } + + @Override + public void write(byte[] b, int off, int l) { + System.arraycopy(b, off, array, c, l); + c += l; + } + })); + } catch (IOException ioe) { + // should never happen because we write to a byte array + throw new RuntimeException("unexpected error while serializing to a byte array"); + } + return new ImmutableBytesWritable(array); } - private ConciseSet bytesToSet(ImmutableBytesWritable bytes) { - if (bytes.get() == null || bytes.getLength() == 0) { - return new ConciseSet(); - } else { - IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer(); - int[] words = new int[intBuffer.capacity()]; - intBuffer.get(words); - return new ConciseSet(words, false); + private RoaringBitmap bytesToSet(final ImmutableBytesWritable bytes) { + // converting a byte array to a bitmap can be expected to be expensive, hopefully this is not a common operation! + RoaringBitmap set = new RoaringBitmap(); + if ((bytes.get() != null) && (bytes.getLength() > 0)) { + // here we could use an ImmutableRoaringBitmap and just "map" it. + // instead, we do a full deserialization + // Note: we deserializing a Roaring bitmap, there is no need to know the length, the format is self-describing + try { + set.deserialize(new java.io.DataInputStream(new java.io.InputStream() { + byte[] array = bytes.get(); + int c = bytes.getOffset(); + + @Override + public int read() { + return array[c++] & 0xff; + } + + @Override + public int read(byte b[]) { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int l) { + System.arraycopy(array, c, b, off, l); + c += l; + return l; + } + })); + } catch (IOException ioe) { + // should never happen because we read from a byte array + throw new RuntimeException("unexpected error while deserializing from a byte array"); + } } + return set; } @Override diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java index 77eb422..48f36f0 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java @@ -20,7 +20,7 @@ package org.apache.kylin.invertedindex.index; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import it.uniroma3.mat.extendedset.intset.ConciseSet; +import org.roaringbitmap.RoaringBitmap; /** * @author yangli9 @@ -36,7 +36,7 @@ public interface ColumnValueContainer { // works only after closeForChange() void getValueAt(int i, ImmutableBytesWritable valueBytes); - ConciseSet getBitMap(Integer startId, Integer endId); + RoaringBitmap getBitMap(Integer startId, Integer endId); int getMaxValueId(); diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java index e301e8d..95fd710 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java @@ -28,7 +28,7 @@ import org.apache.kylin.dict.Dictionary; import com.ning.compress.lzf.LZFDecoder; import com.ning.compress.lzf.LZFEncoder; -import it.uniroma3.mat.extendedset.intset.ConciseSet; +import org.roaringbitmap.RoaringBitmap; /** * @author yangli9 @@ -63,8 +63,8 @@ public class CompressedValueContainer implements ColumnValueContainer { } @Override - public ConciseSet getBitMap(Integer startId, Integer endId) { - ConciseSet ret = new ConciseSet(); + public RoaringBitmap getBitMap(Integer startId, Integer endId) { + RoaringBitmap ret = new RoaringBitmap(); int nullId = Dictionary.NULL_ID[valueLen]; if (startId == null && endId == null) { diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java index cccd503..2a53864 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java @@ -22,7 +22,8 @@ import java.util.Iterator; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import it.uniroma3.mat.extendedset.intset.ConciseSet; +import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.IntIterator; /** * Within a partition (per timestampGranularity), records are further sliced @@ -75,7 +76,7 @@ public class Slice implements Iterable, Comparable { return containers[col]; } - public Iterator iterateWithBitmap(final ConciseSet resultBitMap) { + public Iterator iterateWithBitmap(final RoaringBitmap resultBitMap) { if (resultBitMap == null) { return this.iterator(); } else { @@ -83,27 +84,20 @@ public class Slice implements Iterable, Comparable { final ImmutableBytesWritable temp = new ImmutableBytesWritable(); return new Iterator() { - int i = 0; - int iteratedCount = 0; - int resultSize = resultBitMap.size(); + IntIterator iter = resultBitMap.getIntIterator(); @Override public boolean hasNext() { - return iteratedCount < resultSize; + return iter.hasNext(); } @Override public RawTableRecord next() { - while (!resultBitMap.contains(i)) { - i++; - } + int i = iter.next(); for (int col = 0; col < nColumns; col++) { containers[col].getValueAt(i, temp); rec.setValueBytes(col, temp); } - iteratedCount++; - i++; - return rec; } diff --git a/pom.xml b/pom.xml index c614478..270a6ac 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,8 @@ 2.9.1 2.7.1 1.0.3 - 1.3.4 + + [0.5.4,) 9.2.7.v20150116 2.3 @@ -376,10 +377,15 @@ ${compress-lzf.version} + org.roaringbitmap + RoaringBitmap + ${roaring.version} + + org.apache.curator diff --git a/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java b/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java index 0387abc..3bcee29 100644 --- a/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java +++ b/storage/src/main/java/org/apache/kylin/storage/filter/BitMapFilterEvaluator.java @@ -26,7 +26,7 @@ import org.apache.kylin.metadata.filter.LogicalTupleFilter; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.model.TblColRef; -import it.uniroma3.mat.extendedset.intset.ConciseSet; +import org.roaringbitmap.RoaringBitmap; /** * @author yangli9 @@ -39,7 +39,7 @@ public class BitMapFilterEvaluator { public static interface BitMapProvider { /** return records whose specified column having specified value */ - ConciseSet getBitMap(TblColRef col, Integer startId, Integer endId); + RoaringBitmap getBitMap(TblColRef col, Integer startId, Integer endId); /** return the size of the group */ int getRecordCount(); @@ -58,7 +58,7 @@ public class BitMapFilterEvaluator { * @param filter * @return a set of records that match the filter; or null if filter is null or unable to evaluate */ - public ConciseSet evaluate(TupleFilter filter) { + public RoaringBitmap evaluate(TupleFilter filter) { if (filter == null) return null; @@ -71,7 +71,7 @@ public class BitMapFilterEvaluator { return null; // unable to evaluate } - private ConciseSet evalCompare(CompareTupleFilter filter) { + private RoaringBitmap evalCompare(CompareTupleFilter filter) { switch (filter.getOperator()) { case ISNULL: return evalCompareIsNull(filter); @@ -98,86 +98,89 @@ public class BitMapFilterEvaluator { } } - private ConciseSet evalCompareLT(CompareTupleFilter filter) { + private RoaringBitmap evalCompareLT(CompareTupleFilter filter) { int id = Dictionary.stringToDictId(filter.getFirstValue()); return collectRange(filter.getColumn(), null, id - 1); } - private ConciseSet evalCompareLTE(CompareTupleFilter filter) { + private RoaringBitmap evalCompareLTE(CompareTupleFilter filter) { int id = Dictionary.stringToDictId(filter.getFirstValue()); return collectRange(filter.getColumn(), null, id); } - private ConciseSet evalCompareGT(CompareTupleFilter filter) { + private RoaringBitmap evalCompareGT(CompareTupleFilter filter) { int id = Dictionary.stringToDictId(filter.getFirstValue()); return collectRange(filter.getColumn(), id + 1, null); } - private ConciseSet evalCompareGTE(CompareTupleFilter filter) { + private RoaringBitmap evalCompareGTE(CompareTupleFilter filter) { int id = Dictionary.stringToDictId(filter.getFirstValue()); return collectRange(filter.getColumn(), id, null); } - private ConciseSet collectRange(TblColRef column, Integer startId, Integer endId) { + private RoaringBitmap collectRange(TblColRef column, Integer startId, Integer endId) { return provider.getBitMap(column, startId, endId); } - private ConciseSet evalCompareEqual(CompareTupleFilter filter) { + private RoaringBitmap evalCompareEqual(CompareTupleFilter filter) { int id = Dictionary.stringToDictId(filter.getFirstValue()); - ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id, id); + RoaringBitmap bitMap = provider.getBitMap(filter.getColumn(), id, id); if (bitMap == null) return null; - return bitMap.clone(); // NOTE the clone() to void messing provider's cache + return bitMap.clone(); // NOTE the clone() to void messing provider's cache // If the object is immutable, this is likely wasteful } - private ConciseSet evalCompareNotEqual(CompareTupleFilter filter) { - ConciseSet set = evalCompareEqual(filter); + private RoaringBitmap evalCompareNotEqual(CompareTupleFilter filter) { + RoaringBitmap set = evalCompareEqual(filter); not(set); dropNull(set, filter); return set; } - private ConciseSet evalCompareIn(CompareTupleFilter filter) { - ConciseSet set = new ConciseSet(); + private RoaringBitmap evalCompareIn(CompareTupleFilter filter) { + RoaringBitmap set = new RoaringBitmap(); + java.util.ArrayList buffer = new java.util.ArrayList(); + // an iterator would be better than an ArrayList, but there is + // the convention that says that if one bitmap is null, we return null... for (String value : filter.getValues()) { int id = Dictionary.stringToDictId(value); - ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id, id); + RoaringBitmap bitMap = provider.getBitMap(filter.getColumn(), id, id); if (bitMap == null) return null; - set.addAll(bitMap); + buffer.add(bitMap); } - return set; + return RoaringBitmap.or(buffer.iterator()); } - private ConciseSet evalCompareNotIn(CompareTupleFilter filter) { - ConciseSet set = evalCompareIn(filter); + private RoaringBitmap evalCompareNotIn(CompareTupleFilter filter) { + RoaringBitmap set = evalCompareIn(filter); not(set); dropNull(set, filter); return set; } - private void dropNull(ConciseSet set, CompareTupleFilter filter) { + private void dropNull(RoaringBitmap set, CompareTupleFilter filter) { if (set == null) return; - ConciseSet nullSet = evalCompareIsNull(filter); - set.removeAll(nullSet); + RoaringBitmap nullSet = evalCompareIsNull(filter); + set.andNot(nullSet); } - private ConciseSet evalCompareIsNull(CompareTupleFilter filter) { - ConciseSet bitMap = provider.getBitMap(filter.getColumn(), null, null); + private RoaringBitmap evalCompareIsNull(CompareTupleFilter filter) { + RoaringBitmap bitMap = provider.getBitMap(filter.getColumn(), null, null); if (bitMap == null) return null; - return bitMap.clone(); // NOTE the clone() to void messing provider's cache + return bitMap.clone(); // NOTE the clone() to void messing provider's cache // If the object is immutable, this is likely wasteful } - private ConciseSet evalCompareIsNotNull(CompareTupleFilter filter) { - ConciseSet set = evalCompareIsNull(filter); + private RoaringBitmap evalCompareIsNotNull(CompareTupleFilter filter) { + RoaringBitmap set = evalCompareIsNull(filter); not(set); return set; } - private ConciseSet evalLogical(LogicalTupleFilter filter) { + private RoaringBitmap evalLogical(LogicalTupleFilter filter) { List children = filter.getChildren(); switch (filter.getOperator()) { @@ -192,51 +195,51 @@ public class BitMapFilterEvaluator { } } - private ConciseSet evalLogicalAnd(List children) { - ConciseSet set = new ConciseSet(); - not(set); - - for (TupleFilter c : children) { - ConciseSet t = evaluate(c); + private RoaringBitmap evalLogicalAnd(List children) { + int i = 0; + RoaringBitmap answer = null; + // we identify the first non-null + for(; (i < children.size()) && (answer == null); ++i) { + RoaringBitmap t = evaluate(children.get(i)); if (t == null) - continue; // because it's AND - - set.retainAll(t); + continue; // because it's AND // following convention + answer = t; } - return set; + // then we compute the intersections + for(; i < children.size(); ++i) { + RoaringBitmap t = evaluate(children.get(i)); + if (t == null) + continue; // because it's AND // following convention + answer.and(t); + } + if(answer == null) + answer = new RoaringBitmap(); + return answer; } - private ConciseSet evalLogicalOr(List children) { - ConciseSet set = new ConciseSet(); - + private RoaringBitmap evalLogicalOr(List children) { + java.util.ArrayList buffer = new java.util.ArrayList(); + // could be done with iterator but there is the rule if that if there is a null, then we need to return null for (TupleFilter c : children) { - ConciseSet t = evaluate(c); + RoaringBitmap t = evaluate(c); if (t == null) - return null; // because it's OR - - set.addAll(t); + return null; // because it's OR // following convention + buffer.add(t); } - return set; + return RoaringBitmap.or(buffer.iterator()); } - private ConciseSet evalLogicalNot(List children) { - ConciseSet set = evaluate(children.get(0)); + private RoaringBitmap evalLogicalNot(List children) { + RoaringBitmap set = evaluate(children.get(0)); not(set); return set; } - private void not(ConciseSet set) { + private void not(RoaringBitmap set) { if (set == null) return; - - set.add(provider.getRecordCount()); - set.complement(); + set.flip(0,provider.getRecordCount()); } - public static void main(String[] args) { - ConciseSet s = new ConciseSet(); - s.add(5); - s.complement(); - System.out.println(s); - } + } diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java index c20b09b..d55361b 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java @@ -51,7 +51,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; import com.google.protobuf.Service; -import it.uniroma3.mat.extendedset.intset.ConciseSet; +import org.roaringbitmap.RoaringBitmap; /** * Created by honma on 11/7/14. @@ -128,7 +128,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop EndpointAggregationCache aggCache = new EndpointAggregationCache(aggregators); IIProtos.IIResponse.Builder responseBuilder = IIProtos.IIResponse.newBuilder(); for (Slice slice : slices) { - ConciseSet result = null; + RoaringBitmap result = null; if (filter != null) { result = new BitMapFilterEvaluator(new SliceBitMapProvider(slice, type)).evaluate(filter.getFilter()); } @@ -158,7 +158,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop private IIProtos.IIResponse getNonAggregatedResponse(Iterable slices, CoprocessorFilter filter, CoprocessorRowType type) { IIProtos.IIResponse.Builder responseBuilder = IIProtos.IIResponse.newBuilder(); for (Slice slice : slices) { - ConciseSet result = null; + RoaringBitmap result = null; if (filter != null) { result = new BitMapFilterEvaluator(new SliceBitMapProvider(slice, type)).evaluate(filter.getFilter()); } diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java index 76adadd..c5bee40 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/SliceBitMapProvider.java @@ -23,7 +23,7 @@ import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.storage.filter.BitMapFilterEvaluator; import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType; -import it.uniroma3.mat.extendedset.intset.ConciseSet; +import org.roaringbitmap.RoaringBitmap; /** * Created by Hongbin Ma(Binmahone) on 11/24/14. @@ -41,7 +41,7 @@ public class SliceBitMapProvider implements BitMapFilterEvaluator.BitMapProvider } @Override - public ConciseSet getBitMap(TblColRef col, Integer startId, Integer endId) { + public RoaringBitmap getBitMap(TblColRef col, Integer startId, Integer endId) { return slice.getColumnValueContainer(type.getColIndexByTblColRef(col)).getBitMap(startId, endId); } diff --git a/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java b/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java index ca6a957..2b3799b 100644 --- a/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java +++ b/storage/src/test/java/org/apache/kylin/storage/filter/BitMapFilterEvaluatorTest.java @@ -21,6 +21,7 @@ package org.apache.kylin.storage.filter; import static org.junit.Assert.assertEquals; import java.util.ArrayList; +import java.util.Iterator; import org.apache.kylin.dict.Dictionary; import org.apache.kylin.metadata.filter.ColumnTupleFilter; @@ -36,7 +37,7 @@ import org.apache.kylin.storage.filter.BitMapFilterEvaluator.BitMapProvider; import org.junit.Test; import com.google.common.collect.Lists; -import it.uniroma3.mat.extendedset.intset.ConciseSet; +import org.roaringbitmap.RoaringBitmap; public class BitMapFilterEvaluatorTest { @@ -65,41 +66,40 @@ public class BitMapFilterEvaluatorTest { private static final int REC_COUNT = 10; @Override - public ConciseSet getBitMap(TblColRef col, Integer startId, Integer endId) { + public RoaringBitmap getBitMap(final TblColRef col, Integer startId, Integer endId) { if (!col.equals(colA)) return null; // i-th record has value ID i, and last record has value null if (startId == null && endId == null) { //entry for getting null value - ConciseSet s = new ConciseSet(); - s.add(getRecordCount() - 1); - return s; + return RoaringBitmap.bitmapOf(getRecordCount() - 1); } - int start = 0; - int end = MAX_ID; - if (startId != null) { - start = startId; - } - if (endId != null) { - end = endId; - } - - ConciseSet ret = new ConciseSet(); - for (int i = start; i <= end; ++i) { - ConciseSet temp = getBitMap(col, i); - ret.addAll(temp); - } - return ret; + final int start = (startId != null) ? startId : 0; + final int end = (endId != null) ? endId : MAX_ID; + return RoaringBitmap.or( + new Iterator() { + int i = start; + @Override + public boolean hasNext() { + return i <= end; + } + + @Override + public RoaringBitmap next() { + return getBitMap(col, i++); + } + } + ); } - public ConciseSet getBitMap(TblColRef col, int valueId) { + public RoaringBitmap getBitMap(TblColRef col, int valueId) { if (!col.equals(colA)) return null; // i-th record has value ID i, and last record has value null - ConciseSet bitMap = new ConciseSet(); + RoaringBitmap bitMap = new RoaringBitmap(); if (valueId < 0 || valueId > getMaxValueId(col)) // null bitMap.add(getRecordCount() - 1); else @@ -121,7 +121,7 @@ public class BitMapFilterEvaluatorTest { BitMapFilterEvaluator eval = new BitMapFilterEvaluator(new MockBitMapProivder()); ArrayList basicFilters = Lists.newArrayList(); - ArrayList basicResults = Lists.newArrayList(); + ArrayList basicResults = Lists.newArrayList(); public BitMapFilterEvaluatorTest() { basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL)); @@ -167,8 +167,8 @@ public class BitMapFilterEvaluatorTest { for (int i = 0; i < basicFilters.size(); i++) { for (int j = 0; j < basicFilters.size(); j++) { LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j)); - ConciseSet r = basicResults.get(i).clone(); - r.retainAll(basicResults.get(j)); + RoaringBitmap r = basicResults.get(i).clone(); + r.and(basicResults.get(j)); assertEquals(r, eval.evaluate(f)); } } @@ -179,8 +179,8 @@ public class BitMapFilterEvaluatorTest { for (int i = 0; i < basicFilters.size(); i++) { for (int j = 0; j < basicFilters.size(); j++) { LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j)); - ConciseSet r = basicResults.get(i).clone(); - r.addAll(basicResults.get(j)); + RoaringBitmap r = basicResults.get(i).clone(); + r.or(basicResults.get(j)); assertEquals(r, eval.evaluate(f)); } } @@ -223,11 +223,8 @@ public class BitMapFilterEvaluatorTest { return new ConstantTupleFilter(idToStr(id)); } - public static ConciseSet set(int... ints) { - ConciseSet set = new ConciseSet(); - for (int i : ints) - set.add(i); - return set; + public static RoaringBitmap set(int... ints) { + return RoaringBitmap.bitmapOf(ints); } public static String idToStr(int id) {