diff --git pom.xml pom.xml index efe5e3a..1bd0ab4 100644 --- pom.xml +++ pom.xml @@ -126,7 +126,7 @@ 4.2.5 4.2.5 1.9.2 - 0.3.2 + 0.9.2 5.5.1 3.0.1 7.6.0.v20120127 diff --git ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectInput.java ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectInput.java index 75f66a4..1824b3a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectInput.java +++ ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectInput.java @@ -20,35 +20,20 @@ import java.io.IOException; import java.io.ObjectInput; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.lazy.LazyLong; /** * An ObjectInput that allows for conversion from an List of LongWritable * to an EWAH-compressed bitmap. */ public class BitmapObjectInput implements ObjectInput { - Iterator bufferIter; - List buffer; + + private int index; + private long[] values; - public BitmapObjectInput() { - buffer = new ArrayList(); - bufferIter = buffer.iterator(); - } - - public BitmapObjectInput(List l) { - readFromList(l); - } - - public void readFromList(List l) { - buffer = l; - bufferIter = buffer.iterator(); + public BitmapObjectInput reset(long[] values) { + this.index = 0; + this.values = values; + return this; } @Override @@ -124,15 +109,10 @@ public void readFully(byte[] arg0, int arg1, int arg2) throws IOException { @Override public int readInt() throws IOException { - if (bufferIter.hasNext()) { - LongObjectInspector loi = PrimitiveObjectInspectorFactory.writableLongObjectInspector; - Long l = PrimitiveObjectInspectorUtils.getLong(bufferIter.next(), loi); - return l.intValue(); - //return bufferIter.next().intValue(); - } - else { - throw new IOException(); + if (index < values.length) { + return (int)values[index++]; } + throw new IOException(); } @Override @@ -142,15 +122,10 @@ public String readLine() throws IOException { @Override public long readLong() throws IOException { - //LongObjectInspector loi = PrimitiveObjectInspectorFactory.writableLongObjectInspector; - if (bufferIter.hasNext()) { - LongObjectInspector loi = PrimitiveObjectInspectorFactory.writableLongObjectInspector; - return PrimitiveObjectInspectorUtils.getLong(bufferIter.next(), loi); - //return bufferIter.next(); - } - else { - throw new IOException(); + if (index < values.length) { + return values[index++]; } + throw new IOException(); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectOutput.java ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectOutput.java index 7a4b9d8..f776c2c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectOutput.java +++ ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapObjectOutput.java @@ -22,19 +22,16 @@ import java.io.ObjectOutput; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; /** * An ObjectOutput that allows conversion from an EWAH-compressed bitmap * to an List of LongWritable. */ public class BitmapObjectOutput implements ObjectOutput { - ArrayList buffer = new ArrayList(); + + List buffer = new ArrayList(); - public List list() { + public List list() { return buffer; } @@ -105,12 +102,12 @@ public void writeFloat(float v) throws IOException { @Override public void writeInt(int v) throws IOException { - buffer.add(new LongWritable(v)); + buffer.add((long)v); } @Override public void writeLong(long v) throws IOException { - buffer.add(new LongWritable(v)); + buffer.add(v); } @Override @@ -123,4 +120,7 @@ public void writeUTF(String s) throws IOException { throw new UnsupportedOperationException(); } + public void clear() { + buffer.clear(); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java index 58ea3ba..251cc4e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java @@ -18,36 +18,37 @@ package org.apache.hadoop.hive.ql.udf.generic; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectInput; +import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectOutput; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; -import javaewah.EWAHCompressedBitmap; +import com.googlecode.javaewah.EWAHCompressedBitmap; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; -import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; -import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectInput; -import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectOutput; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.io.LongWritable; /** * An abstract class for a UDF that performs a binary operation between two EWAH-compressed bitmaps. * For example: Bitmap OR and AND operations between two EWAH-compressed bitmaps. */ abstract public class AbstractGenericUDFEWAHBitmapBop extends GenericUDF { - protected final ArrayList ret = new ArrayList(); - private transient ObjectInspector b1OI; + + private transient ListObjectInspector b1OI; + private transient ListObjectInspector b2OI; private final String name; + private final EWAHCompressedBitmap bitmap1 = new EWAHCompressedBitmap(); + private final EWAHCompressedBitmap bitmap2 = new EWAHCompressedBitmap(); + + private final EWAHCompressedBitmap container = new EWAHCompressedBitmap(); + + private final BitmapObjectInput bitmapIn = new BitmapObjectInput(); + private final BitmapObjectOutput bitmapOut = new BitmapObjectOutput(); + AbstractGenericUDFEWAHBitmapBop(String name) { this.name = name; } @@ -58,99 +59,28 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen throw new UDFArgumentLengthException( "The function " + name + "(b1, b2) takes exactly 2 arguments"); } + b1OI = EWAHUtils.getPrimitiveListOI(arguments[0], name); + b2OI = EWAHUtils.getPrimitiveListOI(arguments[1], name); - if (arguments[0].getCategory().equals(Category.LIST)) { - b1OI = (ListObjectInspector) arguments[0]; - } else { - throw new UDFArgumentTypeException(0, "\"" - + Category.LIST.toString().toLowerCase() - + "\" is expected at function " + name + ", but \"" - + arguments[0].getTypeName() + "\" is found"); - } - - if (!arguments[1].getCategory().equals(Category.LIST)) { - throw new UDFArgumentTypeException(1, "\"" - + Category.LIST.toString().toLowerCase() - + "\" is expected at function " + name + ", but \"" - + arguments[1].getTypeName() + "\" is found"); - - } return ObjectInspectorFactory .getStandardListObjectInspector(PrimitiveObjectInspectorFactory - .writableLongObjectInspector); + .javaLongObjectInspector); } protected abstract EWAHCompressedBitmap bitmapBop( - EWAHCompressedBitmap bitmap1, EWAHCompressedBitmap bitmap2); - + EWAHCompressedBitmap bitmap1, EWAHCompressedBitmap bitmap2, EWAHCompressedBitmap container); + @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { assert (arguments.length == 2); + Object b1 = arguments[0].get(); Object b2 = arguments[1].get(); - EWAHCompressedBitmap bitmap1 = wordArrayToBitmap(b1); - EWAHCompressedBitmap bitmap2 = wordArrayToBitmap(b2); - - EWAHCompressedBitmap bitmapAnd = bitmapBop(bitmap1, bitmap2); - - BitmapObjectOutput bitmapObjOut = new BitmapObjectOutput(); - try { - bitmapAnd.writeExternal(bitmapObjOut); - } catch (IOException e) { - throw new RuntimeException(e); - } - ret.clear(); - List retList = bitmapToWordArray(bitmapAnd); - for (LongWritable l : retList) { - ret.add(l); - } - return ret; - } - - protected EWAHCompressedBitmap wordArrayToBitmap(Object b) { - ListObjectInspector lloi = (ListObjectInspector) b1OI; - int length = lloi.getListLength(b); - ArrayList bitmapArray = new ArrayList(); - for (int i = 0; i < length; i++) { - long l = PrimitiveObjectInspectorUtils.getLong( - lloi.getListElement(b, i), - (PrimitiveObjectInspector) lloi.getListElementObjectInspector()); - bitmapArray.add(new LongWritable(l)); - } - - BitmapObjectInput bitmapObjIn = new BitmapObjectInput(bitmapArray); - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - try { - bitmap.readExternal(bitmapObjIn); - } catch (IOException e) { - throw new RuntimeException(e); - } - return bitmap; - } + EWAHCompressedBitmap arg1 = EWAHUtils.wordArrayToBitmap(bitmap1, bitmapIn, b1OI, b1); + EWAHCompressedBitmap arg2 = EWAHUtils.wordArrayToBitmap(bitmap2, bitmapIn, b2OI, b2); - protected List bitmapToWordArray(EWAHCompressedBitmap bitmap) { - BitmapObjectOutput bitmapObjOut = new BitmapObjectOutput(); - try { - bitmap.writeExternal(bitmapObjOut); - } catch (IOException e) { - throw new RuntimeException(e); - } - return bitmapObjOut.list(); - } - - @Override - public String getDisplayString(String[] children) { - StringBuilder sb = new StringBuilder(); - sb.append(name); - sb.append("("); - for (int i = 0; i < children.length; i++) { - sb.append(children[i]); - if (i + 1 != children.length) { - sb.append(","); - } - } - sb.append(")"); - return sb.toString(); + EWAHCompressedBitmap bitmap = bitmapBop(arg1, arg2, container); + return EWAHUtils.bitmapToWordArray(bitmap, bitmapOut); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/EWAHUtils.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/EWAHUtils.java new file mode 100644 index 0000000..0c7da0f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/EWAHUtils.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import com.googlecode.javaewah.EWAHCompressedBitmap; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectInput; +import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectOutput; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +import java.io.IOException; +import java.util.List; + +public class EWAHUtils { + + public static ListObjectInspector getPrimitiveListOI(ObjectInspector argument, String name) + throws UDFArgumentTypeException { + if (!argument.getCategory().equals(ObjectInspector.Category.LIST)) { + throw new UDFArgumentTypeException(0, "\"" + + ObjectInspector.Category.LIST.toString().toLowerCase() + + "\" is expected at function " + name + ", but \"" + + argument.getTypeName() + "\" is found"); + } + ListObjectInspector b1OI = (ListObjectInspector) argument; + if (!(b1OI.getListElementObjectInspector() instanceof PrimitiveObjectInspector)) { + throw new UDFArgumentTypeException(0, "\"" + + "Primitive type elements are expected at function " + name + ", but \"" + + b1OI.getListElementObjectInspector().getTypeName() + "\" is found"); + } + return b1OI; + } + + public static EWAHCompressedBitmap wordArrayToBitmap(EWAHCompressedBitmap bitmap, + BitmapObjectInput bitmapIn, ListObjectInspector lloi, Object b) { + long[] values = new long[lloi.getListLength(b)]; + for (int i = 0; i < values.length; i++) { + values[i] = PrimitiveObjectInspectorUtils.getLong( + lloi.getListElement(b, i), + (PrimitiveObjectInspector) lloi.getListElementObjectInspector()); + } + bitmap.clear(); + try { + bitmap.readExternal(bitmapIn.reset(values)); + } catch (IOException e) { + throw new RuntimeException(e); + } + return bitmap; + } + + public static List bitmapToWordArray( + EWAHCompressedBitmap bitmap, BitmapObjectOutput bitmapOut) { + bitmapOut.clear(); + try { + bitmap.writeExternal(bitmapOut); + } catch (IOException e) { + throw new RuntimeException(e); + } + return bitmapOut.list(); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java index e4b412e..6dc97e9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java @@ -17,11 +17,7 @@ */ package org.apache.hadoop.hive.ql.udf.generic; -import java.io.IOException; -import java.util.ArrayList; - -import javaewah.EWAHCompressedBitmap; - +import com.googlecode.javaewah.EWAHCompressedBitmap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.Description; @@ -30,16 +26,15 @@ import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectOutput; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.util.StringUtils; @@ -72,12 +67,17 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) public static class GenericUDAFEWAHBitmapEvaluator extends GenericUDAFEvaluator { // For PARTIAL1 and COMPLETE: ObjectInspectors for original data - private PrimitiveObjectInspector inputOI; + private transient PrimitiveObjectInspector inputOI; // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations // (lists of bitmaps) - private transient StandardListObjectInspector loi; - private transient StandardListObjectInspector internalMergeOI; + private transient ListObjectInspector internalMergeOI; + + private final EWAHCompressedBitmap bitmap1 = new EWAHCompressedBitmap(); + private final EWAHCompressedBitmap container = new EWAHCompressedBitmap(); + + private final BitmapObjectInput bitmapIn = new BitmapObjectInput(); + private final BitmapObjectOutput bitmapOut = new BitmapObjectOutput(); @Override public ObjectInspector init(Mode m, ObjectInspector[] parameters) @@ -85,28 +85,19 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) super.init(m, parameters); // init output object inspectors // The output of a partial aggregation is a list - if (m == Mode.PARTIAL1) { + if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) { inputOI = (PrimitiveObjectInspector) parameters[0]; - return ObjectInspectorFactory - .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector); } else if (m == Mode.PARTIAL2 || m == Mode.FINAL) { - internalMergeOI = (StandardListObjectInspector) parameters[0]; - inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector; - loi = (StandardListObjectInspector) ObjectInspectorFactory - .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector); - return loi; - } else { // Mode.COMPLETE, ie. no map-side aggregation, requires ordering - inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector; - loi = (StandardListObjectInspector) ObjectInspectorFactory - .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector); - return loi; + internalMergeOI = EWAHUtils.getPrimitiveListOI(parameters[0], "ewah_bitmap"); } + return ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaLongObjectInspector); } /** class for storing the current partial result aggregation */ @AggregationType(estimable = true) static class BitmapAgg extends AbstractAggregationBuffer { - EWAHCompressedBitmap bitmap; + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); @Override public int estimate() { return bitmap.sizeInBytes(); @@ -115,8 +106,7 @@ public int estimate() { @Override public void reset(AggregationBuffer agg) throws HiveException { - - ((BitmapAgg) agg).bitmap = new EWAHCompressedBitmap(); + ((BitmapAgg) agg).bitmap.clear(); } @Override @@ -129,59 +119,38 @@ public AggregationBuffer getNewAggregationBuffer() throws HiveException { @Override public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { - assert (parameters.length == 1); - Object p = parameters[0]; - if (p != null) { - BitmapAgg myagg = (BitmapAgg) agg; - try { - int row = PrimitiveObjectInspectorUtils.getInt(p, inputOI); - addBitmap(row, myagg); - } catch (NumberFormatException e) { - LOG.warn(getClass().getSimpleName() + " " + - StringUtils.stringifyException(e)); - } + assert (parameters.length == 1); + Object p = parameters[0]; + if (p != null) { + BitmapAgg myagg = (BitmapAgg) agg; + try { + int row = PrimitiveObjectInspectorUtils.getInt(p, inputOI); + addBitmap(row, myagg); + } catch (NumberFormatException e) { + LOG.warn(getClass().getSimpleName() + " " + + StringUtils.stringifyException(e)); } + } } - @Override public Object terminate(AggregationBuffer agg) throws HiveException { - BitmapAgg myagg = (BitmapAgg) agg; - - BitmapObjectOutput bitmapObjOut = new BitmapObjectOutput(); - try { - myagg.bitmap.writeExternal(bitmapObjOut); - } catch (IOException e) { - throw new RuntimeException(e); - } - return bitmapObjOut.list(); + BitmapAgg myagg = (BitmapAgg) agg; + return EWAHUtils.bitmapToWordArray(myagg.bitmap, bitmapOut); } @Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { BitmapAgg myagg = (BitmapAgg) agg; - ArrayList partialResult = (ArrayList) internalMergeOI.getList(partial); - BitmapObjectInput bitmapObjIn = new BitmapObjectInput(partialResult); - EWAHCompressedBitmap partialBitmap = new EWAHCompressedBitmap(); - try { - partialBitmap.readExternal(bitmapObjIn); - } catch (IOException e) { - throw new RuntimeException(e); - } - myagg.bitmap = myagg.bitmap.or(partialBitmap); + myagg.bitmap = myagg.bitmap.or( + EWAHUtils.wordArrayToBitmap(bitmap1, bitmapIn, internalMergeOI, partial)); } @Override public Object terminatePartial(AggregationBuffer agg) throws HiveException { BitmapAgg myagg = (BitmapAgg) agg; - BitmapObjectOutput bitmapObjOut = new BitmapObjectOutput(); - try { - myagg.bitmap.writeExternal(bitmapObjOut); - } catch (IOException e) { - throw new RuntimeException(e); - } - return bitmapObjOut.list(); + return EWAHUtils.bitmapToWordArray(myagg.bitmap, bitmapOut); } private void addBitmap(int newRow, BitmapAgg myagg) { diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java index e3fb558..7c8f8f3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java @@ -179,8 +179,20 @@ public abstract Object evaluate(DeferredObject[] arguments) /** * Get the String to be displayed in explain. */ - public abstract String getDisplayString(String[] children); - + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append(getUdfName()); + sb.append('('); + for (int i = 0; i < children.length; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(children[i]); + } + sb.append(')'); + return sb.toString(); + } + /** * Close GenericUDF. * This is only called in runtime of MapRedTask. diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapAnd.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapAnd.java index 7838b54..1341525 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapAnd.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapAnd.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.ql.udf.generic; -import javaewah.EWAHCompressedBitmap; +import com.googlecode.javaewah.EWAHCompressedBitmap; import org.apache.hadoop.hive.ql.exec.Description; @@ -36,7 +36,8 @@ public GenericUDFEWAHBitmapAnd() { @Override protected EWAHCompressedBitmap bitmapBop( - EWAHCompressedBitmap bitmap1, EWAHCompressedBitmap bitmap2) { - return bitmap1.and(bitmap2); + EWAHCompressedBitmap bitmap1, EWAHCompressedBitmap bitmap2, EWAHCompressedBitmap container) { + bitmap1.andToContainer(bitmap2, container); + return container; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java index 4a14a65..ceb9be9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java @@ -18,92 +18,46 @@ package org.apache.hadoop.hive.ql.udf.generic; -import java.io.IOException; -import java.util.ArrayList; - -import javaewah.EWAHCompressedBitmap; - +import com.googlecode.javaewah.EWAHCompressedBitmap; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; -import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.index.bitmap.BitmapObjectInput; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.io.BooleanWritable; -import org.apache.hadoop.io.LongWritable; @Description(name = "ewah_bitmap_empty", value = "_FUNC_(bitmap) - " + "Predicate that tests whether an EWAH-compressed bitmap is all zeros ") public class GenericUDFEWAHBitmapEmpty extends GenericUDF { - private transient ObjectInspector bitmapOI; - private transient BooleanObjectInspector boolOI; - -@Override -public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { - if (arguments.length != 1) { - throw new UDFArgumentLengthException( - "The function EWAH_BITMAP_EMPTY(b) takes exactly 1 argument"); - } - - if (arguments[0].getCategory().equals(Category.LIST)) { - bitmapOI = (ListObjectInspector) arguments[0]; - } else { - throw new UDFArgumentTypeException(0, "\"" - + Category.LIST.toString().toLowerCase() - + "\" is expected at function EWAH_BITMAP_EMPTY, but \"" - + arguments[0].getTypeName() + "\" is found"); - } - - boolOI = PrimitiveObjectInspectorFactory.writableBooleanObjectInspector; - return boolOI; + + private static final String name = "EWAH_BITMAP_EMPTY"; + + private transient ListObjectInspector bitmapOI; + + private final EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + private final BitmapObjectInput bitmapIn = new BitmapObjectInput(); + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentLengthException( + "The function EWAH_BITMAP_EMPTY(b) takes exactly 1 argument"); + } + bitmapOI = EWAHUtils.getPrimitiveListOI(arguments[0], name); + return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector; } @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { assert (arguments.length == 1); Object b = arguments[0].get(); - - ListObjectInspector lloi = (ListObjectInspector) bitmapOI; - int length = lloi.getListLength(b); - ArrayList bitmapArray = new ArrayList(); - for (int i = 0; i < length; i++) { - long l = PrimitiveObjectInspectorUtils.getLong( - lloi.getListElement(b, i), - (PrimitiveObjectInspector) lloi.getListElementObjectInspector()); - bitmapArray.add(new LongWritable(l)); - } - - BitmapObjectInput bitmapObjIn = new BitmapObjectInput(bitmapArray); - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - try { - bitmap.readExternal(bitmapObjIn); - } catch (IOException e) { - throw new RuntimeException(e); - } - - // Add return true only if bitmap is all zeros. - return new BooleanWritable(!bitmap.iterator().hasNext()); + return EWAHUtils.wordArrayToBitmap(bitmap, bitmapIn, bitmapOI, b).isEmpty(); } - @Override - public String getDisplayString(String[] children) { - StringBuilder sb = new StringBuilder(); - sb.append("EWAH_BITMAP_EMPTY("); - for (int i = 0; i < children.length; i++) { - sb.append(children[i]); - if (i + 1 != children.length) { - sb.append(","); - } - } - sb.append(")"); - return sb.toString(); + public String getUdfName() { + return name; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapOr.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapOr.java index d438f82..f1e3386 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapOr.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapOr.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.ql.udf.generic; -import javaewah.EWAHCompressedBitmap; +import com.googlecode.javaewah.EWAHCompressedBitmap; import org.apache.hadoop.hive.ql.exec.Description; @@ -36,7 +36,8 @@ public GenericUDFEWAHBitmapOr() { @Override protected EWAHCompressedBitmap bitmapBop( - EWAHCompressedBitmap bitmap1, EWAHCompressedBitmap bitmap2) { - return bitmap1.or(bitmap2); + EWAHCompressedBitmap bitmap1, EWAHCompressedBitmap bitmap2, EWAHCompressedBitmap container) { + bitmap1.orToContainer(bitmap2, container); + return container; } } diff --git ql/src/test/queries/clientpositive/index_bitmap2.q ql/src/test/queries/clientpositive/index_bitmap2.q index 89fbe76..208fda5 100644 --- ql/src/test/queries/clientpositive/index_bitmap2.q +++ ql/src/test/queries/clientpositive/index_bitmap2.q @@ -23,7 +23,7 @@ SELECT t.bucketname as `_bucketname`, COLLECT_SET(t.offset) AS `_offsets` FROM WHERE key = 0 AND NOT EWAH_BITMAP_EMPTY(`_bitmaps`) UNION ALL SELECT `_bucketname` AS bucketname, `_offset` AS offset FROM default__src_src2_index__ - WHERE value = "val2" AND NOT EWAH_BITMAP_EMPTY(`_bitmaps`)) t + WHERE value = "val_2" AND NOT EWAH_BITMAP_EMPTY(`_bitmaps`)) t GROUP BY t.bucketname; SET hive.index.blockfilter.file=${system:test.tmp.dir}/index_result; diff --git ql/src/test/queries/clientpositive/udf_bitmap_and.q ql/src/test/queries/clientpositive/udf_bitmap_and.q index ed7711c..9bce867 100644 --- ql/src/test/queries/clientpositive/udf_bitmap_and.q +++ ql/src/test/queries/clientpositive/udf_bitmap_and.q @@ -1,13 +1,13 @@ set hive.fetch.task.conversion=more; -select ewah_bitmap_and(array(13,2,4,8589934592,4096,0), array(13,2,4,8589934592,4096,0)) from src tablesample (1 rows); -select ewah_bitmap_and(array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0)) from src tablesample (1 rows); +select ewah_bitmap_and(array(13,2,8589934592,4096,0), array(13,2,8589934592,4096,0)) from src tablesample (1 rows); +select ewah_bitmap_and(array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0)) from src tablesample (1 rows); drop table bitmap_test; create table bitmap_test (a array, b array); insert overwrite table bitmap_test -select array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0) from src tablesample (10 rows); +select array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0) from src tablesample (10 rows); select ewah_bitmap_and(a,b) from bitmap_test; diff --git ql/src/test/queries/clientpositive/udf_bitmap_empty.q ql/src/test/queries/clientpositive/udf_bitmap_empty.q index 142b248..c2376ae 100644 --- ql/src/test/queries/clientpositive/udf_bitmap_empty.q +++ ql/src/test/queries/clientpositive/udf_bitmap_empty.q @@ -1,5 +1,5 @@ set hive.fetch.task.conversion=more; -select ewah_bitmap_empty(array(13,2,4,8589934592,0,0)) from src tablesample (1 rows); +select ewah_bitmap_empty(array(13,2,8589934592,0,0)) from src tablesample (1 rows); -select ewah_bitmap_empty(array(13,2,4,8589934592,4096,0)) from src tablesample (1 rows); +select ewah_bitmap_empty(array(13,2,8589934592,4096,0)) from src tablesample (1 rows); diff --git ql/src/test/queries/clientpositive/udf_bitmap_or.q ql/src/test/queries/clientpositive/udf_bitmap_or.q index 00785b7..b81905b 100644 --- ql/src/test/queries/clientpositive/udf_bitmap_or.q +++ ql/src/test/queries/clientpositive/udf_bitmap_or.q @@ -1,13 +1,13 @@ set hive.fetch.task.conversion=more; -select ewah_bitmap_or(array(13,2,4,8589934592,4096,0), array(13,2,4,8589934592,4096,0)) from src tablesample (1 rows); -select ewah_bitmap_or(array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0)) from src tablesample (1 rows); +select ewah_bitmap_or(array(13,2,8589934592,4096,0), array(13,2,8589934592,4096,0)) from src tablesample (1 rows); +select ewah_bitmap_or(array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0)) from src tablesample (1 rows); drop table bitmap_test; create table bitmap_test (a array, b array); insert overwrite table bitmap_test -select array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0) from src tablesample (10 rows); +select array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0) from src tablesample (10 rows); select ewah_bitmap_or(a,b) from bitmap_test; diff --git ql/src/test/results/clientpositive/index_bitmap2.q.out ql/src/test/results/clientpositive/index_bitmap2.q.out index 73c5b90..f006070 100644 --- ql/src/test/results/clientpositive/index_bitmap2.q.out +++ ql/src/test/results/clientpositive/index_bitmap2.q.out @@ -86,7 +86,7 @@ SELECT t.bucketname as `_bucketname`, COLLECT_SET(t.offset) AS `_offsets` FROM WHERE key = 0 AND NOT EWAH_BITMAP_EMPTY(`_bitmaps`) UNION ALL SELECT `_bucketname` AS bucketname, `_offset` AS offset FROM default__src_src2_index__ - WHERE value = "val2" AND NOT EWAH_BITMAP_EMPTY(`_bitmaps`)) t + WHERE value = "val_2" AND NOT EWAH_BITMAP_EMPTY(`_bitmaps`)) t GROUP BY t.bucketname PREHOOK: type: QUERY PREHOOK: Input: default@default__src_src1_index__ @@ -98,7 +98,7 @@ SELECT t.bucketname as `_bucketname`, COLLECT_SET(t.offset) AS `_offsets` FROM WHERE key = 0 AND NOT EWAH_BITMAP_EMPTY(`_bitmaps`) UNION ALL SELECT `_bucketname` AS bucketname, `_offset` AS offset FROM default__src_src2_index__ - WHERE value = "val2" AND NOT EWAH_BITMAP_EMPTY(`_bitmaps`)) t + WHERE value = "val_2" AND NOT EWAH_BITMAP_EMPTY(`_bitmaps`)) t GROUP BY t.bucketname POSTHOOK: type: QUERY POSTHOOK: Input: default@default__src_src1_index__ diff --git ql/src/test/results/clientpositive/index_bitmap3.q.out ql/src/test/results/clientpositive/index_bitmap3.q.out index 599bf3a..13968a3 100644 --- ql/src/test/results/clientpositive/index_bitmap3.q.out +++ ql/src/test/results/clientpositive/index_bitmap3.q.out @@ -113,35 +113,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_src1_index__ - Statistics: Num rows: 500 Data size: 46311 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 45311 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((key = 0) and _bucketname is not null) and _offset is not null) (type: boolean) - Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5709 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _bucketname (type: string), _offset (type: bigint), _bitmaps (type: array) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5709 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5709 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: array) TableScan alias: default__src_src2_index__ - Statistics: Num rows: 500 Data size: 48311 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 47311 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((value = 'val_0') and _bucketname is not null) and _offset is not null) (type: boolean) - Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5961 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _bucketname (type: string), _offset (type: bigint), _bitmaps (type: array) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5961 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5961 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: array) Reduce Operator Tree: Join Operator @@ -151,20 +151,20 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: bigint) 1 _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 69 Data size: 6418 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 69 Data size: 6279 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not EWAH_BITMAP_EMPTY(EWAH_BITMAP_AND(_col2,_col5))) (type: boolean) - Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE + predicate: (not EWAH_BITMAP_EMPTY(org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapAnd(_col2,_col5))) (type: boolean) + Statistics: Num rows: 35 Data size: 3185 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3185 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: collect_set(_col1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3185 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -180,7 +180,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3185 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: array) Reduce Operator Tree: Group By Operator @@ -188,14 +188,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1547 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: array) outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1547 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1547 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/index_bitmap_auto.q.out ql/src/test/results/clientpositive/index_bitmap_auto.q.out index 81c1795..cf5aaed 100644 --- ql/src/test/results/clientpositive/index_bitmap_auto.q.out +++ ql/src/test/results/clientpositive/index_bitmap_auto.q.out @@ -132,35 +132,35 @@ STAGE PLANS: Map Operator Tree: TableScan alias: default__src_src1_index__ - Statistics: Num rows: 500 Data size: 46311 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 45311 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((key = 0) and _bucketname is not null) and _offset is not null) (type: boolean) - Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5709 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _bucketname (type: string), _offset (type: bigint), _bitmaps (type: array) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5709 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 63 Data size: 5835 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5709 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: array) TableScan alias: default__src_src2_index__ - Statistics: Num rows: 500 Data size: 48311 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 47311 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((value = 'val_0') and _bucketname is not null) and _offset is not null) (type: boolean) - Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5961 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _bucketname (type: string), _offset (type: bigint), _bitmaps (type: array) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5961 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: bigint) sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) - Statistics: Num rows: 63 Data size: 6087 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 5961 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: array) Reduce Operator Tree: Join Operator @@ -170,20 +170,20 @@ STAGE PLANS: 0 _col0 (type: string), _col1 (type: bigint) 1 _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col5 - Statistics: Num rows: 69 Data size: 6418 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 69 Data size: 6279 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (not EWAH_BITMAP_EMPTY(EWAH_BITMAP_AND(_col2,_col5))) (type: boolean) - Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE + predicate: (not EWAH_BITMAP_EMPTY(org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapAnd(_col2,_col5))) (type: boolean) + Statistics: Num rows: 35 Data size: 3185 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3185 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: collect_set(_col1) keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3185 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -199,7 +199,7 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 35 Data size: 3255 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 35 Data size: 3185 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: array) Reduce Operator Tree: Group By Operator @@ -207,14 +207,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1547 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: array) outputColumnNames: _col0, _col1 - Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1547 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 17 Data size: 1581 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 1547 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/udf_bitmap_and.q.out ql/src/test/results/clientpositive/udf_bitmap_and.q.out index 8c93398..e763c80 100644 --- ql/src/test/results/clientpositive/udf_bitmap_and.q.out +++ ql/src/test/results/clientpositive/udf_bitmap_and.q.out @@ -1,21 +1,21 @@ -PREHOOK: query: select ewah_bitmap_and(array(13,2,4,8589934592,4096,0), array(13,2,4,8589934592,4096,0)) from src tablesample (1 rows) +PREHOOK: query: select ewah_bitmap_and(array(13,2,8589934592,4096,0), array(13,2,8589934592,4096,0)) from src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select ewah_bitmap_and(array(13,2,4,8589934592,4096,0), array(13,2,4,8589934592,4096,0)) from src tablesample (1 rows) +POSTHOOK: query: select ewah_bitmap_and(array(13,2,8589934592,4096,0), array(13,2,8589934592,4096,0)) from src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -[13,2,4,8589934592,4096,0] -PREHOOK: query: select ewah_bitmap_and(array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0)) from src tablesample (1 rows) +[13,2,8589934592,4096,0] +PREHOOK: query: select ewah_bitmap_and(array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0)) from src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select ewah_bitmap_and(array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0)) from src tablesample (1 rows) +POSTHOOK: query: select ewah_bitmap_and(array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0)) from src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -[13,1,4,2,0] +[13,2,8589934592,0,0] PREHOOK: query: drop table bitmap_test PREHOOK: type: DROPTABLE POSTHOOK: query: drop table bitmap_test @@ -29,12 +29,12 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@bitmap_test PREHOOK: query: insert overwrite table bitmap_test -select array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0) from src tablesample (10 rows) +select array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0) from src tablesample (10 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@bitmap_test POSTHOOK: query: insert overwrite table bitmap_test -select array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0) from src tablesample (10 rows) +select array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0) from src tablesample (10 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@bitmap_test @@ -48,16 +48,16 @@ POSTHOOK: query: select ewah_bitmap_and(a,b) from bitmap_test POSTHOOK: type: QUERY POSTHOOK: Input: default@bitmap_test #### A masked pattern was here #### -[13,1,4,2,0] -[13,1,4,2,0] -[13,1,4,2,0] -[13,1,4,2,0] -[13,1,4,2,0] -[13,1,4,2,0] -[13,1,4,2,0] -[13,1,4,2,0] -[13,1,4,2,0] -[13,1,4,2,0] +[13,2,8589934592,0,0] +[13,2,8589934592,0,0] +[13,2,8589934592,0,0] +[13,2,8589934592,0,0] +[13,2,8589934592,0,0] +[13,2,8589934592,0,0] +[13,2,8589934592,0,0] +[13,2,8589934592,0,0] +[13,2,8589934592,0,0] +[13,2,8589934592,0,0] PREHOOK: query: drop table bitmap_test PREHOOK: type: DROPTABLE PREHOOK: Input: default@bitmap_test diff --git ql/src/test/results/clientpositive/udf_bitmap_empty.q.out ql/src/test/results/clientpositive/udf_bitmap_empty.q.out index ca96e78..d25da08 100644 --- ql/src/test/results/clientpositive/udf_bitmap_empty.q.out +++ ql/src/test/results/clientpositive/udf_bitmap_empty.q.out @@ -1,17 +1,17 @@ -PREHOOK: query: select ewah_bitmap_empty(array(13,2,4,8589934592,0,0)) from src tablesample (1 rows) +PREHOOK: query: select ewah_bitmap_empty(array(13,2,8589934592,0,0)) from src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select ewah_bitmap_empty(array(13,2,4,8589934592,0,0)) from src tablesample (1 rows) +POSTHOOK: query: select ewah_bitmap_empty(array(13,2,8589934592,0,0)) from src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### true -PREHOOK: query: select ewah_bitmap_empty(array(13,2,4,8589934592,4096,0)) from src tablesample (1 rows) +PREHOOK: query: select ewah_bitmap_empty(array(13,2,8589934592,4096,0)) from src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select ewah_bitmap_empty(array(13,2,4,8589934592,4096,0)) from src tablesample (1 rows) +POSTHOOK: query: select ewah_bitmap_empty(array(13,2,8589934592,4096,0)) from src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/udf_bitmap_or.q.out ql/src/test/results/clientpositive/udf_bitmap_or.q.out index 43521da..e45da8a 100644 --- ql/src/test/results/clientpositive/udf_bitmap_or.q.out +++ ql/src/test/results/clientpositive/udf_bitmap_or.q.out @@ -1,21 +1,21 @@ -PREHOOK: query: select ewah_bitmap_or(array(13,2,4,8589934592,4096,0), array(13,2,4,8589934592,4096,0)) from src tablesample (1 rows) +PREHOOK: query: select ewah_bitmap_or(array(13,2,8589934592,4096,0), array(13,2,8589934592,4096,0)) from src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select ewah_bitmap_or(array(13,2,4,8589934592,4096,0), array(13,2,4,8589934592,4096,0)) from src tablesample (1 rows) +POSTHOOK: query: select ewah_bitmap_or(array(13,2,8589934592,4096,0), array(13,2,8589934592,4096,0)) from src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -[13,2,4,8589934592,4096,0] -PREHOOK: query: select ewah_bitmap_or(array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0)) from src tablesample (1 rows) +[13,2,8589934592,4096,0] +PREHOOK: query: select ewah_bitmap_or(array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0)) from src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select ewah_bitmap_or(array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0)) from src tablesample (1 rows) +POSTHOOK: query: select ewah_bitmap_or(array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0)) from src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -[13,2,4,8589934592,4224,0] +[13,2,8589934592,4224,0] PREHOOK: query: drop table bitmap_test PREHOOK: type: DROPTABLE POSTHOOK: query: drop table bitmap_test @@ -29,12 +29,12 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@bitmap_test PREHOOK: query: insert overwrite table bitmap_test -select array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0) from src tablesample (10 rows) +select array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0) from src tablesample (10 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@bitmap_test POSTHOOK: query: insert overwrite table bitmap_test -select array(13,2,4,8589934592,4096,0), array(8,2,4,8589934592,128,0) from src tablesample (10 rows) +select array(13,2,8589934592,4096,0), array(8,2,8589934592,128,0) from src tablesample (10 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@bitmap_test @@ -48,16 +48,16 @@ POSTHOOK: query: select ewah_bitmap_or(a,b) from bitmap_test POSTHOOK: type: QUERY POSTHOOK: Input: default@bitmap_test #### A masked pattern was here #### -[13,2,4,8589934592,4224,0] -[13,2,4,8589934592,4224,0] -[13,2,4,8589934592,4224,0] -[13,2,4,8589934592,4224,0] -[13,2,4,8589934592,4224,0] -[13,2,4,8589934592,4224,0] -[13,2,4,8589934592,4224,0] -[13,2,4,8589934592,4224,0] -[13,2,4,8589934592,4224,0] -[13,2,4,8589934592,4224,0] +[13,2,8589934592,4224,0] +[13,2,8589934592,4224,0] +[13,2,8589934592,4224,0] +[13,2,8589934592,4224,0] +[13,2,8589934592,4224,0] +[13,2,8589934592,4224,0] +[13,2,8589934592,4224,0] +[13,2,8589934592,4224,0] +[13,2,8589934592,4224,0] +[13,2,8589934592,4224,0] PREHOOK: query: drop table bitmap_test PREHOOK: type: DROPTABLE PREHOOK: Input: default@bitmap_test