diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/DeltaIntegralCompressor.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/DeltaIntegralCompressor.java new file mode 100644 index 0000000..87d4b21 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/DeltaIntegralCompressor.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.hadoop.hive.contrib.ubercompressor.InputReader; +import org.apache.hadoop.hive.contrib.ubercompressor.OutputWriter; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitInputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitOutputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.impl.BitInputStreamImpl; +import org.apache.hadoop.hive.contrib.ubercompressor.io.impl.BitOutputStreamImpl; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +/** + * TwoSidedIntegralCompressor. Takes a 1-sided AbstractIntegralCompressor and + * diffs monotonic input values + */ +class DeltaIntegralCompressor extends IntegralCompressor { + + private static class MyOutputWriter extends IntegralOutputWriter { + + MyOutputWriter(BitOutputStream out, ObjectInspector oi, LongWriter writer) throws IOException { + super(out, oi, writer); + } + + private boolean firstTime = true; + private long prev = 0; + private byte order = 0; // 0 - unknown, +1 ascending, -1 descending + private int numRepeats = 0; + + @Override + protected void doWrite(long value) throws IOException { + // FIXME input validation + if (firstTime) { + firstTime = false; + prev = value; + } else { + if (order == 0) { + if (value == prev) { + ++numRepeats; + } else { + if (value > prev) { + order = 1; + mOut.writeBit((byte) 0x01); + } else { + order = -1; + mOut.writeBit((byte) 0x00); + } + mOut.writeBits(prev, 64); + for (int i = 0; i < numRepeats; ++i) { + super.doWrite(0); + } + super.doWrite(order * (value - prev)); + prev = value; + } + } else { + super.doWrite(order * (value - prev)); + prev = value; + } + } + } + + @Override + public long close(boolean closeUnderlying) throws IOException { + if (!firstTime) { + if (order == 0) { + mOut.writeBit((byte) 0x01); + mOut.writeBits(prev, 64); + for (int i = 0; i < numRepeats; ++i) { + super.doWrite(0); + } + } + } + return super.close(closeUnderlying); + } + + } + + private static class MyInputReader extends IntegralInputReader { + + private boolean firstTime = true; + private long prev = 0; + private byte order = 0; // 0 - EOF, +1 ascending, -1 descending + + MyInputReader(BitInputStream in, LongReader reader) throws IOException { + super(in, reader); + try { + if (mIn.readBit() == 1) { + order = 1; + } else { + order = -1; + } + } catch (EOFException e) { + } + } + + @Override + protected long doRead() throws EOFException, IOException { + if (order == 0) { + throw new EOFException(); + } + if (firstTime) { + firstTime = false; + long rc = mIn.readBits(64); + prev = rc; + return rc; + } else { + long delta = super.doRead(); + prev = prev + order * delta; + return prev; + } + } + + } + + public DeltaIntegralCompressor(LongHelper helper) { + super(helper); + } + + @Override + public OutputWriter createOutputWriter(OutputStream out, ObjectInspector oi) throws IOException { + BitOutputStream bout = new BitOutputStreamImpl(out); + return new MyOutputWriter(bout, oi, mHelper.createWriter(bout)); + } + + @Override + public InputReader createInputReader(InputStream in, long numBits) throws IOException { + BitInputStream bin = new BitInputStreamImpl(in); + return new MyInputReader(bin, mHelper.createReader(bin)); + } + +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCoder.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCoder.java new file mode 100644 index 0000000..1aa9e0d --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCoder.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.IOException; + +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitInputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitOutputStream; + +class EliasGammaCoder implements LongHelper { + + protected class MyWriter implements LongWriter { + + private BitOutputStream mOut; + + MyWriter(BitOutputStream out) { + mOut = out; + } + + @Override + public void writeLong(long value) throws IOException { + if (value < 0) { + throw new IllegalArgumentException("Gamma encode called with -ve value:" + value); + } else { + if (value == Long.MAX_VALUE) { + throw new IllegalArgumentException("Gamma encode called with long.max value:" + value); + } + } + ++value; + long tmp = value; + int bitlen = 0; + while (tmp > 0) { + tmp >>>= 1; + ++bitlen; + } + mOut.writeBits(0, bitlen - 1); + mOut.writeBits(value, bitlen); + } + + @Override + public void close() { + } + } + + public class MyReader implements LongReader { + + private BitInputStream mIn; + + MyReader(BitInputStream in) { + mIn = in; + } + + @Override + public long readLong() throws IOException { + int bitlen = 0; + while (mIn.readBit() != 1) { + ++bitlen; + } + long v = mIn.readBits(bitlen) | (0x01L << bitlen); + return v - 1; + } + + @Override + public void close() throws IOException { + } + + } + + @Override + public LongWriter createWriter(BitOutputStream out) { + return new MyWriter(out); + } + + @Override + public LongReader createReader(BitInputStream in) { + return new MyReader(in); + } +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressor.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressor.java new file mode 100644 index 0000000..8cfd23b --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressor.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +public class EliasGammaCompressor extends IntegralCompressor { + + public EliasGammaCompressor() { + super(new EliasGammaCoder()); + } +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressor2S.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressor2S.java new file mode 100644 index 0000000..366b6af --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressor2S.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +public class EliasGammaCompressor2S extends TwoSidedIntegralCompressor { + + public EliasGammaCompressor2S() { + super(new EliasGammaCoder()); + } +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressorD.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressorD.java new file mode 100644 index 0000000..a1588ab --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressorD.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +public class EliasGammaCompressorD extends DeltaIntegralCompressor { + + public EliasGammaCompressorD() { + super(new EliasGammaCoder()); + } +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressorD2S.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressorD2S.java new file mode 100644 index 0000000..d875005 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/EliasGammaCompressorD2S.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +public class EliasGammaCompressorD2S extends TwoSidedDeltaIntegralCompressor { + + public EliasGammaCompressorD2S() { + super(new EliasGammaCoder()); + } +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/IntegralCompressor.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/IntegralCompressor.java new file mode 100644 index 0000000..e84ab67 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/IntegralCompressor.java @@ -0,0 +1,244 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.hadoop.hive.contrib.ubercompressor.InputReader; +import org.apache.hadoop.hive.contrib.ubercompressor.OutputWriter; +import org.apache.hadoop.hive.contrib.ubercompressor.TypeSpecificCompressor; +import org.apache.hadoop.hive.contrib.ubercompressor.dsalg.Tuple; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitInputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitOutputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.impl.BitInputStreamImpl; +import org.apache.hadoop.hive.contrib.ubercompressor.io.impl.BitOutputStreamImpl; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; + +interface LongWriter { + /* + * Typically should be able to write a non-negative value inc. MAX_VALUE. May defer the actual + * write but should flush all deferred writes on close. + */ + void writeLong(long value) throws IOException; + + /* + * do not close the underlying stream + */ + void close() throws IOException;; +} + +interface LongReader { + /* + * Typically should be able to read a non-negative value + */ + long readLong() throws EOFException, IOException;; + + /* + * do not close the underlying stream + */ + void close() throws IOException; +} + +interface LongHelper { + LongWriter createWriter(BitOutputStream out); + + LongReader createReader(BitInputStream in); +} + +class IntegralCompressor implements TypeSpecificCompressor { + + static class IntegralOutputWriter implements OutputWriter { + + protected BitOutputStream mOut; + private BooleanObjectInspector mBooleanOI; + private ByteObjectInspector mByteOI; + private ShortObjectInspector mShortOI; + private IntObjectInspector mIntOI; + private LongObjectInspector mLongOI; + private PrimitiveCategory mPC; + private LongWriter mWriter; + + IntegralOutputWriter(BitOutputStream out, ObjectInspector oi, LongWriter writer) + throws IOException { + mOut = out; + Category c = oi.getCategory(); + switch (c) { + case PRIMITIVE: + mPC = ((PrimitiveObjectInspector) oi).getPrimitiveCategory(); + switch (mPC) { + case BOOLEAN: + mBooleanOI = (BooleanObjectInspector) oi; + break; + case BYTE: + mByteOI = (ByteObjectInspector) oi; + break; + case SHORT: + mShortOI = (ShortObjectInspector) oi; + break; + case INT: + mIntOI = (IntObjectInspector) oi; + break; + case LONG: + mLongOI = (LongObjectInspector) oi; + break; + default: + throw new IllegalArgumentException( + "DummyIntegerCompressor passed invalid objectInspector of type:" + oi.getTypeName()); + } + break; + default: + throw new IllegalArgumentException( + "DummyIntegerCompressor passed invalid objectInspector of type:" + oi.getTypeName()); + } + mOut.writeBits(mPC.ordinal(), 8); + mWriter = writer; + } + + @Override + public void write(Object t) throws IOException { + long value = 0; + switch (mPC) { + case BOOLEAN: + value = (mBooleanOI.get(t)) ? 1 : 0; + break; + case BYTE: + value = mByteOI.get(t); + break; + case SHORT: + value = mShortOI.get(t); + break; + case INT: + value = mIntOI.get(t); + break; + case LONG: + value = mLongOI.get(t); + break; + } + doWrite(value); + } + + protected void doWrite(long value) throws IOException { + mWriter.writeLong(value); + } + + @Override + public long close(boolean closeUnderlying) throws IOException { + mWriter.close(); + return mOut.close(closeUnderlying); + } + } + + static class IntegralInputReader implements InputReader { + + private PrimitiveCategory mPC; + protected BitInputStream mIn; + private AbstractPrimitiveJavaObjectInspector mOI; + private Tuple mRV = new Tuple(); + private LongReader mReader; + + IntegralInputReader(BitInputStream in, LongReader reader) throws IOException { + mIn = in; + mPC = PrimitiveCategory.values()[(int) mIn.readBits(8)]; + mOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(mPC); + mReader = reader; + } + + + @Override + public ObjectInspector getObjectInspector() { + return mOI; + } + + @Override + public Tuple read() throws IOException { + try { + long value = doRead(); + Object rv = null; + switch (mPC) { + case BOOLEAN: + if (value == 0) { + rv = Boolean.FALSE; + } else { + rv = Boolean.TRUE; + } + break; + case BYTE: + rv = Byte.valueOf((byte) value); + break; + case SHORT: + rv = Short.valueOf((short) value); + break; + case INT: + rv = Integer.valueOf((int) value); + break; + case LONG: + rv = Long.valueOf(value); + break; + } + mRV.setFirst(Boolean.FALSE); + mRV.setSecond(rv); + return mRV; + } catch (EOFException e) { + mRV.setFirst(Boolean.TRUE); + mRV.setSecond(null); + return mRV; + } + } + + protected long doRead() throws EOFException, IOException { + return mReader.readLong(); + } + + @Override + public void close(boolean closeUnderLying) throws IOException { + mReader.close(); + mIn.close(closeUnderLying); + } + } + + protected LongHelper mHelper; + + public IntegralCompressor(LongHelper helper) { + mHelper = helper; + } + + @Override + public OutputWriter createOutputWriter(OutputStream out, ObjectInspector oi) throws IOException { + BitOutputStream bout = new BitOutputStreamImpl(out); + return new IntegralOutputWriter(bout, oi, mHelper.createWriter(bout)); + } + + @Override + public InputReader createInputReader(InputStream in, long numBits) throws IOException { + BitInputStream bin = new BitInputStreamImpl(in); + return new IntegralInputReader(bin, mHelper.createReader(bin)); + } +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TwoSidedDeltaIntegralCompressor.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TwoSidedDeltaIntegralCompressor.java new file mode 100644 index 0000000..93de099 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TwoSidedDeltaIntegralCompressor.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.hadoop.hive.contrib.ubercompressor.InputReader; +import org.apache.hadoop.hive.contrib.ubercompressor.OutputWriter; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitInputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitOutputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.impl.BitInputStreamImpl; +import org.apache.hadoop.hive.contrib.ubercompressor.io.impl.BitOutputStreamImpl; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + + +/** + * TwoSidedIntegralCompressor. Takes a 1-sided AbstractIntegralCompressor and + * diffs monotonic input values + */ +class TwoSidedDeltaIntegralCompressor extends IntegralCompressor { + + private static class MyOutputWriter extends TwoSidedIntegralCompressor.MyOutputWriter { + + MyOutputWriter(BitOutputStream out, ObjectInspector oi, LongWriter writer) throws IOException { + super(out, oi, writer); + } + + private boolean firstTime = true; + private long prev = 0; + + @Override + protected void doWrite(long value) throws IOException { + // FIXME input validation + if (firstTime) { + firstTime = false; + mOut.writeBits(value, 64); + prev = value; + } else { + super.doWrite(value - prev); + prev = value; + } + } + } + + private static class MyInputReader extends TwoSidedIntegralCompressor.MyInputReader { + + MyInputReader(BitInputStream in, LongReader reader) throws IOException { + super(in, reader); + } + + private boolean firstTime = true; + private long prev = 0; + + @Override + protected long doRead() throws EOFException, IOException { + // FIXME validate + if (firstTime) { + firstTime = false; + long rc = mIn.readBits(64); + prev = rc; + return rc; + } else { + long delta = super.doRead(); + prev = prev + delta; + return prev; + } + } + + } + + public TwoSidedDeltaIntegralCompressor(LongHelper helper) { + super(helper); + } + + @Override + public OutputWriter createOutputWriter(OutputStream out, ObjectInspector oi) throws IOException { + BitOutputStream bout = new BitOutputStreamImpl(out); + return new MyOutputWriter(bout, oi, mHelper.createWriter(bout)); + } + + @Override + public InputReader createInputReader(InputStream in, long numBits) throws IOException { + BitInputStream bin = new BitInputStreamImpl(in); + return new MyInputReader(bin, mHelper.createReader(bin)); + } + +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TwoSidedIntegralCompressor.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TwoSidedIntegralCompressor.java new file mode 100644 index 0000000..3f7a116 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TwoSidedIntegralCompressor.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.hadoop.hive.contrib.ubercompressor.InputReader; +import org.apache.hadoop.hive.contrib.ubercompressor.OutputWriter; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitInputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitOutputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.impl.BitInputStreamImpl; +import org.apache.hadoop.hive.contrib.ubercompressor.io.impl.BitOutputStreamImpl; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +/** + * TwoSidedIntegralCompressor. Takes a 1-sided AbstractIntegralCompressor and + * folds 2-sided input values to positive values + */ +class TwoSidedIntegralCompressor extends IntegralCompressor { + + static final long MAX_VALUE = Long.MAX_VALUE / 2; + static final long MIN_VALUE = - MAX_VALUE; + + protected static class MyOutputWriter extends IntegralOutputWriter { + + MyOutputWriter(BitOutputStream out, ObjectInspector oi, LongWriter writer) throws IOException { + super(out, oi, writer); + } + + @Override + protected void doWrite(long value) throws IOException { + if ((value > MAX_VALUE) || (value < MIN_VALUE)) { + throw new IllegalArgumentException("Two-sided compressor cannot fold values beyond range: " + + MIN_VALUE + "-" + MAX_VALUE); + } + long oneSidedPosValue; + if (value >= 0) { + oneSidedPosValue = 2 * value; + } else { + oneSidedPosValue = - 1 - 2 * value; + } + super.doWrite(oneSidedPosValue); + } + } + + protected static class MyInputReader extends IntegralInputReader { + + MyInputReader(BitInputStream in, LongReader reader) throws IOException { + super(in, reader); + } + + @Override + protected long doRead() throws EOFException, IOException { + long rc = super.doRead(); + long i = rc / 2; + if (2 * i != rc) { + i = -i - 1; + } + return i; + } + } + + public TwoSidedIntegralCompressor(LongHelper helper) { + super(helper); + } + + @Override + public OutputWriter createOutputWriter(OutputStream out, ObjectInspector oi) throws IOException { + BitOutputStream bout = new BitOutputStreamImpl(out); + return new MyOutputWriter(bout, oi, mHelper.createWriter(bout)); + } + + @Override + public InputReader createInputReader(InputStream in, long numBits) throws IOException { + BitInputStream bin = new BitInputStreamImpl(in); + return new MyInputReader(bin, mHelper.createReader(bin)); + } + +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCoder.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCoder.java new file mode 100644 index 0000000..f86bad4 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCoder.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.EOFException; +import java.io.IOException; + +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitInputStream; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitOutputStream; + +class UnaryCoder implements LongHelper { + + private static class MyWriter implements LongWriter { + + private BitOutputStream mOut; + + MyWriter(BitOutputStream out) { + mOut = out; + } + + @Override + public void writeLong(long value) throws IOException { + if (value < 0) { + throw new IllegalArgumentException("UnaryCompressor does not support -ve values"); + } + while (value > 0) { + mOut.writeBit((byte) 0x00); + --value; + } + mOut.writeBit((byte) 0x01); + } + + @Override + public void close() { + } + + } + + private static class MyReader implements LongReader { + + private BitInputStream mIn; + + MyReader(BitInputStream in) { + mIn = in; + } + + @Override + public long readLong() throws EOFException, IOException { + long rc = 0; + while (mIn.readBit() != 1) { + ++rc; + } + return rc; + } + + @Override + public void close() throws IOException { + } + + } + + @Override + public LongWriter createWriter(BitOutputStream out) { + return new MyWriter(out); + } + + @Override + public LongReader createReader(BitInputStream in) { + return new MyReader(in); + } + +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressor.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressor.java new file mode 100644 index 0000000..9757a05 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressor.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +public class UnaryCompressor extends IntegralCompressor { + public UnaryCompressor() { + super(new UnaryCoder()); + } +} \ No newline at end of file diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressor2S.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressor2S.java new file mode 100644 index 0000000..0474247 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressor2S.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +public class UnaryCompressor2S extends TwoSidedIntegralCompressor { + public UnaryCompressor2S() { + super(new UnaryCoder()); + } +} \ No newline at end of file diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressorD.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressorD.java new file mode 100644 index 0000000..e4991a5 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressorD.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +public class UnaryCompressorD extends DeltaIntegralCompressor { + public UnaryCompressorD() { + super(new UnaryCoder()); + } +} \ No newline at end of file diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressorD2S.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressorD2S.java new file mode 100644 index 0000000..cc36d8d --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/compressors/UnaryCompressorD2S.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +public class UnaryCompressorD2S extends TwoSidedDeltaIntegralCompressor { + public UnaryCompressorD2S() { + super(new UnaryCoder()); + } +} \ No newline at end of file diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/BitInputStream.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/BitInputStream.java new file mode 100644 index 0000000..dffabbc --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/BitInputStream.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.io; + +import java.io.IOException; + +public interface BitInputStream { + + public byte readBit() throws IOException; + + public long readBits(int numBits) throws IOException; + + public void skipBit() throws IOException; + + public void skipBits(int numBits) throws IOException; + + public void close(boolean closeInner) throws IOException; +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/BitOutputStream.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/BitOutputStream.java new file mode 100644 index 0000000..87f86a4 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/BitOutputStream.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.io; + +import java.io.IOException; + +public interface BitOutputStream { + + public void writeBit(byte bit) throws IOException; + + public void writeBits(long data, int numBits) throws IOException; + + /** + * if true, closes the underlying stream as well + * + * @return the number of bits written, -1 if unknown + * @throws IOException + */ + public long close(boolean closeInner) throws IOException; +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/PeekableBitInputStream.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/PeekableBitInputStream.java new file mode 100644 index 0000000..3b55710 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/PeekableBitInputStream.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.io; + +import java.io.IOException; + +public interface PeekableBitInputStream extends BitInputStream { + public byte peekBit() throws IOException; + + public long peekBits(int numBits) throws IOException; +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/impl/BitInputStreamImpl.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/impl/BitInputStreamImpl.java new file mode 100644 index 0000000..02bb03f --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/impl/BitInputStreamImpl.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.io.impl; + +import java.io.BufferedInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitInputStream; + +/** + * Implementation of the BitInputStream interface + * + */ +public class BitInputStreamImpl implements BitInputStream { + + private BufferedInputStream mIn; + private long mTotalBits = -1; + + private long mTotalReadBits = 0; + + private byte currentReadByte; + private int bitReadIndex = -1; + + public BitInputStreamImpl(InputStream in) { + mIn = new BufferedInputStream(in); + } + + public BitInputStreamImpl(InputStream in, long totalBits) { + mIn = new BufferedInputStream(in); + mTotalBits = totalBits; + } + + + /* + * (non-Javadoc) + * + * @see com.yahoo.datacomp.io.impl.BitOutputStream#readBit() + */ + @Override + public byte readBit() throws IOException { + if (mTotalBits != -1) { + if (mTotalBits <= mTotalReadBits) { + throw new EOFException(); + } + } + if (bitReadIndex < 0) { + readByte(); + } + byte rc = (byte) ((currentReadByte >>> bitReadIndex) & 0x01); + --bitReadIndex; + ++mTotalReadBits; + return rc; + } + + private static final int BITMASK[] = { + 0x00000001, 0x00000003, + 0x00000007, 0x00000000F, + 0x0000001F, 0x0000003F, + 0x0000007F, 0x000000FF}; + + /* + * (non-Javadoc) + * + * @see com.yahoo.datacomp.io.impl.BitOutputStream#readBits(int) + */ + @Override + public long readBits(int numBits) throws IOException { + assert (numBits <= 64); + if (mTotalBits != -1) { + if (mTotalBits < mTotalReadBits + numBits) { + throw new EOFException(); + } + } + long rc = 0; + while (numBits > 0) { + if (bitReadIndex < 0) { + readByte(); + } + if (bitReadIndex + 1 >= numBits) { + int tmp = ((int) (currentReadByte >>> (bitReadIndex + 1 - numBits)) & BITMASK[numBits - 1]); + rc <<= numBits; + rc |= tmp; + bitReadIndex -= numBits; + mTotalReadBits += numBits; + return rc; + } + int numBitsToReadInCurrentByte = bitReadIndex + 1; + int tmp = ((int) currentReadByte) & BITMASK[numBitsToReadInCurrentByte - 1]; + rc <<= numBitsToReadInCurrentByte; + rc |= tmp; + numBits -= numBitsToReadInCurrentByte; + bitReadIndex -= numBitsToReadInCurrentByte; + mTotalReadBits += numBitsToReadInCurrentByte; + } + return rc; + } + + /* + * (non-Javadoc) + * + * @see com.yahoo.datacomp.io.impl.BitOutputStream#skipBit() + */ + @Override + public void skipBit() throws IOException { + if (mTotalBits != -1) { + if (mTotalBits <= mTotalReadBits) { + throw new EOFException(); + } + } + if (bitReadIndex < 0) { + readByte(); + } + --bitReadIndex; + ++mTotalReadBits; + } + + /* + * (non-Javadoc) + * + * @see com.yahoo.datacomp.io.impl.BitOutputStream#skipBits(int) + */ + @Override + public void skipBits(int numBits) throws IOException { + assert (numBits <= 32); + if (mTotalBits != -1) { + if (mTotalBits < mTotalReadBits + numBits) { + throw new EOFException(); + } + } + while (numBits > 0) { + if (bitReadIndex < 0) { + readByte(); + } + if (bitReadIndex + 1 >= numBits) { + bitReadIndex -= numBits; + mTotalReadBits += numBits; + return; + } + int numBitsToReadInCurrentByte = bitReadIndex + 1; + numBits -= numBitsToReadInCurrentByte; + bitReadIndex -= numBitsToReadInCurrentByte; + mTotalReadBits += numBitsToReadInCurrentByte; + } + return; + } + + /* + * (non-Javadoc) + * + * @see com.yahoo.datacomp.io.impl.BitOutputStream#close() + */ + @Override + public void close(boolean closeInner) throws IOException { + if (closeInner) { + mIn.close(); + } + } + + private void readByte() throws IOException { + mIn.reset(); + int rc = mIn.read(); + if (rc == -1) { + throw new EOFException(); + } + currentReadByte = (byte) rc; + bitReadIndex = 7; + } +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/impl/BitOutputStreamImpl.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/impl/BitOutputStreamImpl.java new file mode 100644 index 0000000..7b033e8 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/impl/BitOutputStreamImpl.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.io.impl; + +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.contrib.ubercompressor.io.BitOutputStream; + +public class BitOutputStreamImpl implements BitOutputStream { + + private static final Log LOG = LogFactory.getLog(BitOutputStreamImpl.class); + + private OutputStream mOut; + private byte currentByte; + private int bitIndex = 7; + private long totalBits = 0; + + public BitOutputStreamImpl(OutputStream out) { + mOut = out; + } + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#writeBit(byte) + */ + @Override + public void writeBit(byte bit) throws IOException { + LOG.trace("BitOut: " + Integer.toBinaryString(bit)); + currentByte |= (bit & 0x01) << bitIndex; + --bitIndex; + if (bitIndex < 0) { + mOut.write(currentByte); + bitIndex = 7; + currentByte = 0; + } + ++totalBits; + } + + private static byte BITMASK[] = { 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, (byte) 0xFF }; + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#writeBits(long, int) + */ + @Override + public void writeBits(long data, int numBits) throws IOException { + if (LOG.isTraceEnabled()) { + String s = "0000000000000000000000000000000000000000000000000000000000000000" + + Long.toBinaryString(data); + String s1 = s.substring(s.length() - numBits); + if (s1.length() > 0) LOG.trace("BitsOut: " + s1); + } + assert (numBits <= 64); + totalBits += numBits; + while (numBits > 0) { + if (bitIndex + 1 >= numBits) { + currentByte |= (data & BITMASK[numBits - 1]) << (bitIndex + 1 - numBits); + bitIndex -= numBits; + if (bitIndex < 0) { + mOut.write(currentByte); + bitIndex = 7; + currentByte = 0; + } + return; + } + int numBitsToWriteInCurrentByte = bitIndex + 1; + byte tmp = + (byte) ((data >>> (numBits - numBitsToWriteInCurrentByte)) & BITMASK[numBitsToWriteInCurrentByte - 1]); + currentByte |= tmp; + mOut.write(currentByte); + bitIndex = 7; + currentByte = 0; + numBits -= numBitsToWriteInCurrentByte; + } + } + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#close() + */ + @Override + public long close(boolean closeInner) throws IOException { + if (bitIndex != 7) { + mOut.write(currentByte); + } + mOut.flush(); + if (closeInner) { + mOut.close(); + } + return totalBits; + } + +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/impl/PeekableBitInputStreamImpl.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/impl/PeekableBitInputStreamImpl.java new file mode 100644 index 0000000..1f76105 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/io/impl/PeekableBitInputStreamImpl.java @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.io.impl; + +import java.io.BufferedInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.hadoop.hive.contrib.ubercompressor.io.PeekableBitInputStream; + + +/** + * Implementation of the PeekableBitInputStream interface + * + */ +public class PeekableBitInputStreamImpl implements PeekableBitInputStream { + + private static final int READ_LIMIT = 1024; + private BufferedInputStream mIn; + private long mTotalBits = -1; + + private byte currentPeekedByte; + private int bitPeekIndex = -1; + private long mTotalReadBits = 0; + + private byte currentReadByte; + private int bitReadIndex = -1; + private long mTotalPeekedBits = 0; + + public PeekableBitInputStreamImpl(InputStream in) { + mIn = new BufferedInputStream(in); + mIn.mark(READ_LIMIT); + } + + public PeekableBitInputStreamImpl(InputStream in, long totalBits) { + mIn = new BufferedInputStream(in); + mIn.mark(READ_LIMIT); + mTotalBits = totalBits; + } + + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#readBit() + */ + @Override + public byte readBit() throws IOException { + if (mTotalBits != -1) { + if (mTotalBits <= mTotalReadBits) { + throw new EOFException(); + } + } + if (bitReadIndex < 0) { + readByte(); + } + byte rc = (byte) ((currentReadByte >>> bitReadIndex) & 0x01); + --bitReadIndex; + ++mTotalReadBits; + resetPeek(); + return rc; + } + + private void resetPeek() throws IOException { + currentPeekedByte = currentReadByte; + bitPeekIndex = bitReadIndex; + mIn.reset(); + mTotalPeekedBits = mTotalReadBits; + } + + private static final int BITMASK[] = { + 0x00000001, 0x00000003, + 0x00000007, 0x00000000F, + 0x0000001F, 0x0000003F, + 0x0000007F, 0x000000FF }; + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#readBits(int) + */ + @Override + public long readBits(int numBits) throws IOException { + assert (numBits <= 64); + if (mTotalBits != -1) { + if (mTotalBits < mTotalReadBits + numBits) { + throw new EOFException(); + } + } + long rc = 0; + while (numBits > 0) { + if (bitReadIndex < 0) { + readByte(); + } + if (bitReadIndex + 1 >= numBits) { + int tmp = ((int)(currentReadByte >>> (bitReadIndex + 1 - numBits)) & BITMASK[numBits - 1]); + rc <<= numBits; + rc |= tmp; + bitReadIndex -= numBits; + mTotalReadBits += numBits; + resetPeek(); + return rc; + } + int numBitsToReadInCurrentByte = bitReadIndex + 1; + int tmp = ((int)currentReadByte) & BITMASK[numBitsToReadInCurrentByte - 1]; + rc <<= numBitsToReadInCurrentByte; + rc |= tmp; + numBits -= numBitsToReadInCurrentByte; + bitReadIndex -= numBitsToReadInCurrentByte; + mTotalReadBits += numBitsToReadInCurrentByte; + } + resetPeek(); + return rc; + } + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#skipBit() + */ + @Override + public void skipBit() throws IOException { + if (mTotalBits != -1) { + if (mTotalBits <= mTotalReadBits) { + throw new EOFException(); + } + } + if (bitReadIndex < 0) { + readByte(); + } + --bitReadIndex; + ++mTotalReadBits; + resetPeek(); + } + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#skipBits(int) + */ + @Override + public void skipBits(int numBits) throws IOException { + assert (numBits <= 32); + if (mTotalBits != -1) { + if (mTotalBits < mTotalReadBits + numBits) { + throw new EOFException(); + } + } + while (numBits > 0) { + if (bitReadIndex < 0) { + readByte(); + } + if (bitReadIndex + 1 >= numBits) { + bitReadIndex -= numBits; + mTotalReadBits += numBits; + resetPeek(); + return; + } + int numBitsToReadInCurrentByte = bitReadIndex + 1; + numBits -= numBitsToReadInCurrentByte; + bitReadIndex -= numBitsToReadInCurrentByte; + mTotalReadBits += numBitsToReadInCurrentByte; + } + resetPeek(); + return; + } + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#peekBit() + */ + @Override + public byte peekBit() throws IOException { + if (mTotalBits != -1) { + if (mTotalBits <= mTotalPeekedBits) { + throw new EOFException(); + } + } + if (bitPeekIndex < 0) { + peekByte(); + } + byte rc = (byte) ((currentPeekedByte >>> bitPeekIndex) & 0x01); + --bitPeekIndex; + ++mTotalPeekedBits; + return rc; + } + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#peekBits(int) + */ + @Override + public long peekBits(int numBits) throws IOException { + assert (numBits <= 64); + if (mTotalBits != -1) { + if (mTotalBits < mTotalPeekedBits + numBits) { + throw new EOFException(); + } + } + long rc = 0; + while (numBits > 0) { + if (bitPeekIndex < 0) { + peekByte(); + } + if (bitPeekIndex + 1 >= numBits) { + int tmp = ((int)(currentPeekedByte >>> (bitPeekIndex + 1 - numBits)) & BITMASK[numBits - 1]); + rc <<= numBits; + rc |= tmp; + bitPeekIndex -= numBits; + mTotalReadBits += numBits; + return rc; + } + int numBitsToReadInCurrentByte = bitPeekIndex + 1; + int tmp = ((int)currentPeekedByte) & BITMASK[numBitsToReadInCurrentByte - 1]; + rc <<= numBitsToReadInCurrentByte; + rc |= tmp; + numBits -= numBitsToReadInCurrentByte; + bitPeekIndex -= numBitsToReadInCurrentByte; + mTotalReadBits += numBitsToReadInCurrentByte; + } + return rc; + } + + /* (non-Javadoc) + * @see com.yahoo.datacomp.io.impl.BitOutputStream#close() + */ + @Override + public void close(boolean closeInner) throws IOException { + if (closeInner) { + mIn.close(); + } + } + + private void peekByte() throws IOException { + int rc = mIn.read(); + if (rc == -1) { + throw new EOFException(); + } + currentPeekedByte = (byte) rc; + bitPeekIndex = 7; + } + + private void readByte() throws IOException { + mIn.reset(); + int rc = mIn.read(); + if (rc == -1) { + throw new EOFException(); + } + currentReadByte = (byte) rc; + mIn.mark(READ_LIMIT); + bitReadIndex = 7; + } + + +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/EventHelper.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/EventHelper.java new file mode 100644 index 0000000..8aa0052 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/EventHelper.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.model; + +public interface EventHelper { + +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/EventHelperBijectiveDRV.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/EventHelperBijectiveDRV.java new file mode 100644 index 0000000..fb88700 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/EventHelperBijectiveDRV.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.model; + +public interface EventHelperBijectiveDRV extends EventHelperDRV { + public T getEvent(int drv); +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/EventHelperDRV.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/EventHelperDRV.java new file mode 100644 index 0000000..97309c0 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/EventHelperDRV.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.model; + +public interface EventHelperDRV extends EventHelper { + public int getDrv(T t); +} diff --git contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/impl/IntegerEventHelper.java contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/impl/IntegerEventHelper.java new file mode 100644 index 0000000..bd2cf20 --- /dev/null +++ contrib/src/java/org/apache/hadoop/hive/contrib/ubercompressor/model/impl/IntegerEventHelper.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.model.impl; + +import org.apache.hadoop.hive.contrib.ubercompressor.model.EventHelperBijectiveDRV; + +public class IntegerEventHelper + implements EventHelperBijectiveDRV { + + + @Override + public int getDrv(Integer t) { + if (t == null) { + throw new IllegalArgumentException("IntegerEventHelper does not support null values"); + } + return t.intValue(); + } + + @Override + public Integer getEvent(int drv) { + return Integer.valueOf(drv); + } + +} diff --git contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestBooleanCompressorBase.java contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestBooleanCompressorBase.java new file mode 100644 index 0000000..b271801 --- /dev/null +++ contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestBooleanCompressorBase.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.contrib.ubercompressor.InputReader; +import org.apache.hadoop.hive.contrib.ubercompressor.OutputWriter; +import org.apache.hadoop.hive.contrib.ubercompressor.TypeSpecificCompressor; +import org.apache.hadoop.hive.contrib.ubercompressor.dsalg.Tuple; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; + +public class TestBooleanCompressorBase extends TestCase { + + private TypeSpecificCompressor c; + + TestBooleanCompressorBase(TypeSpecificCompressor c) { + this.c = c; + } + + private boolean[] empty = { + }; + + private boolean[] one = { + true + }; + + private boolean[] two = { + false + }; + + private boolean[] ascending = { + false, + false, + true, + true, + true + }; + + private boolean[] descending = { + true, + true, + true, + false, + false, + false + }; + + private boolean[] zigzag = { + true, + false, + false, + true, + false, + true, + true + }; + + protected void testArray(TypeSpecificCompressor c, boolean[] arr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + BooleanObjectInspector boi = (BooleanObjectInspector) + PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveCategory.BOOLEAN); + OutputWriter ow = c.createOutputWriter(out, boi); + for (boolean b : arr) { + ow.write(b); + } + long numBits = ow.close(true); + byte[] byteArray = out.toByteArray(); + ByteArrayInputStream in = new ByteArrayInputStream(byteArray); + InputReader ir = c.createInputReader(in, numBits); + + for (boolean exp : arr) { + Tuple i = ir.read(); + assertFalse(i.getFirst()); + assertEquals(exp, i.getSecond()); + } + Tuple i = ir.read(); + assertTrue(i.getFirst()); + } + + public void testEmpty() throws IOException { + testArray(c, empty); + } + + public void testOne() throws IOException { + testArray(c, one); + } + + public void testTwo() throws IOException { + testArray(c, two); + } + + public void testAscending() throws IOException { + testArray(c, ascending); + } + + public void testDescending() throws IOException { + testArray(c, descending); + } + + public void testZigZag() throws IOException { + testArray(c, zigzag); + } + +} diff --git contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestByteCompressorBase.java contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestByteCompressorBase.java new file mode 100644 index 0000000..657c906 --- /dev/null +++ contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestByteCompressorBase.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.contrib.ubercompressor.InputReader; +import org.apache.hadoop.hive.contrib.ubercompressor.OutputWriter; +import org.apache.hadoop.hive.contrib.ubercompressor.TypeSpecificCompressor; +import org.apache.hadoop.hive.contrib.ubercompressor.dsalg.Tuple; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; + +public class TestByteCompressorBase extends TestCase { + + private TypeSpecificCompressor c; + + TestByteCompressorBase(TypeSpecificCompressor c) { + this.c = c; + } + + private byte[] nonnegrange = { + 0, + Byte.MAX_VALUE + }; + + private byte[] range = { + Byte.MIN_VALUE, + Byte.MAX_VALUE + }; + + private byte[] nonnegascending = { + 0, + 1, + 2, + 16, + 32, + 64, + Byte.MAX_VALUE + }; + + private byte[] ascending = { + Byte.MIN_VALUE, + -16, + -2, + -1, + 0, + 1, + 2, + 16, + Byte.MAX_VALUE + }; + + private byte[] nonnegdescending = { + Byte.MAX_VALUE, + 64, + 32, + 16, + 2, + 1, + 0, + }; + + private byte[] descending = { + Byte.MAX_VALUE, + 64, + 32, + 16, + 2, + 1, + 0, + -1, + -2, + -16, + -32, + -64, + Byte.MIN_VALUE + }; + + private byte[] nonnegzigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Byte.MAX_VALUE, + 0, + Byte.MAX_VALUE + }; + + private byte[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Byte.MAX_VALUE, + Byte.MIN_VALUE, + Byte.MAX_VALUE + }; + + protected void testArray(TypeSpecificCompressor c, byte[] arr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteObjectInspector boi = (ByteObjectInspector) + PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveCategory.BYTE); + OutputWriter ow = c.createOutputWriter(out, boi); + for (Byte b : arr) { + ow.write(b); + } + long numBits = ow.close(true); + byte[] byteArray = out.toByteArray(); + ByteArrayInputStream in = new ByteArrayInputStream(byteArray); + InputReader ir = c.createInputReader(in, numBits); + + for (Byte exp : arr) { + Tuple i = ir.read(); + assertFalse(i.getFirst()); + assertEquals(exp, i.getSecond()); + } + Tuple i = ir.read(); + assertTrue(i.getFirst()); + } + + public void testNonNegRange() throws IOException { + testArray(c, nonnegrange); + } + + public void testRange() throws IOException { + testArray(c, range); + } + + public void testNonNegAscending() throws IOException { + testArray(c, nonnegascending); + } + + public void testAscending() throws IOException { + testArray(c, ascending); + } + + public void testNonNegDescending() throws IOException { + testArray(c, nonnegdescending); + } + + public void testDescending() throws IOException { + testArray(c, descending); + } + + public void testNonNegZigZag() throws IOException { + testArray(c, nonnegzigzag); + } + + public void testZigZag() throws IOException { + testArray(c, zigzag); + } + +} diff --git contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestEliasGammaCompressor.java contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestEliasGammaCompressor.java new file mode 100644 index 0000000..aad8ebc --- /dev/null +++ contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestEliasGammaCompressor.java @@ -0,0 +1,735 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.IOException; + +import junit.framework.Test; +import junit.framework.TestSuite; + +public class TestEliasGammaCompressor { + public static Test suite() { + TestSuite suite = new TestSuite(TestEliasGammaCompressor.class.getName()); + suite.addTestSuite(TestEliasGammaCompressorForBooleans.class); + suite.addTestSuite(TestEliasGammaCompressorForBytes.class); + suite.addTestSuite(TestEliasGammaCompressorForShorts.class); + suite.addTestSuite(TestEliasGammaCompressorForInts.class); + suite.addTestSuite(TestEliasGammaCompressorForLongs.class); + + suite.addTestSuite(TestEliasGammaCompressor2SForBooleans.class); + suite.addTestSuite(TestEliasGammaCompressor2SForBytes.class); + suite.addTestSuite(TestEliasGammaCompressor2SForShorts.class); + suite.addTestSuite(TestEliasGammaCompressor2SForInts.class); + suite.addTestSuite(TestEliasGammaCompressor2SForLongs.class); + + suite.addTestSuite(TestEliasGammaCompressorDForBooleans.class); + suite.addTestSuite(TestEliasGammaCompressorDForBytes.class); + suite.addTestSuite(TestEliasGammaCompressorDForShorts.class); + suite.addTestSuite(TestEliasGammaCompressorDForInts.class); + suite.addTestSuite(TestEliasGammaCompressorDForLongs.class); + + suite.addTestSuite(TestEliasGammaCompressorD2SForBooleans.class); + suite.addTestSuite(TestEliasGammaCompressorD2SForBytes.class); + suite.addTestSuite(TestEliasGammaCompressorD2SForShorts.class); + suite.addTestSuite(TestEliasGammaCompressorD2SForInts.class); + suite.addTestSuite(TestEliasGammaCompressorD2SForLongs.class); + + return suite; + } + + public static class TestEliasGammaCompressorForBooleans extends TestBooleanCompressorBase { + public TestEliasGammaCompressorForBooleans() { + super(new EliasGammaCompressor()); + } + } + + public static class TestEliasGammaCompressorForBytes extends TestByteCompressorBase { + public TestEliasGammaCompressorForBytes() { + super(new EliasGammaCompressor()); + } + + @Override + public void testRange() throws IOException { + try { + super.testRange(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testAscending() throws IOException { + try { + super.testAscending(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testDescending() throws IOException { + try { + super.testDescending(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestEliasGammaCompressorForShorts extends TestShortCompressorBase { + public TestEliasGammaCompressorForShorts() { + super(new EliasGammaCompressor()); + } + + @Override + public void testRange() throws IOException { + try { + super.testRange(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testAscending() throws IOException { + try { + super.testAscending(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testDescending() throws IOException { + try { + super.testDescending(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestEliasGammaCompressorForInts extends TestIntCompressorBase { + public TestEliasGammaCompressorForInts() { + super(new EliasGammaCompressor()); + } + + @Override + public void testRange() throws IOException { + try { + super.testRange(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testAscending() throws IOException { + try { + super.testAscending(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testDescending() throws IOException { + try { + super.testDescending(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestEliasGammaCompressorForLongs extends TestLongCompressorBase { + public TestEliasGammaCompressorForLongs() { + super(new EliasGammaCompressor()); + } + + @Override + public void testRange() throws IOException { + try { + super.testRange(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testAscending() throws IOException { + try { + super.testAscending(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testDescending() throws IOException { + try { + super.testDescending(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegRange() throws IOException { + try { + super.testNonNegRange(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + public void testSupportedNonNegRange() throws IOException { + long[] nonnegrange = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + Long.MAX_VALUE - 1 + }; + super.testArray(c, nonnegrange); + } + + @Override + public void testNonNegAscending() throws IOException { + long[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + Long.MAX_VALUE - 1 + }; + super.testArray(c, nonnegascending); + } + + @Override + public void testNonNegDescending() throws IOException { + long[] nonnegdescending = { + Long.MAX_VALUE - 1, + Integer.MAX_VALUE, + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0 + }; + super.testArray(c, nonnegdescending); + } + + @Override + public void testNonNegZigZag() throws IOException { + long[] nonnegzigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Long.MAX_VALUE - 1, + 0, + Long.MAX_VALUE - 1 + }; + super.testArray(c, nonnegzigzag); + } + } + + public static class TestEliasGammaCompressor2SForBooleans extends TestBooleanCompressorBase { + public TestEliasGammaCompressor2SForBooleans() { + super(new EliasGammaCompressor2S()); + } + } + + public static class TestEliasGammaCompressor2SForBytes extends TestByteCompressorBase { + public TestEliasGammaCompressor2SForBytes() { + super(new EliasGammaCompressor2S()); + } + } + + public static class TestEliasGammaCompressor2SForShorts extends TestShortCompressorBase { + public TestEliasGammaCompressor2SForShorts() { + super(new EliasGammaCompressor2S()); + } + } + + public static class TestEliasGammaCompressor2SForInts extends TestIntCompressorBase { + public TestEliasGammaCompressor2SForInts() { + super(new EliasGammaCompressor2S()); + } + } + + public static class TestEliasGammaCompressor2SForLongs extends TestLongCompressorBase { + public TestEliasGammaCompressor2SForLongs() { + super(new EliasGammaCompressor2S()); + } + + @Override + public void testNonNegRange() throws IOException { + try { + super.testNonNegRange(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + public void testUnsupportedNonNegRange() throws IOException { + long[] nonnegrange = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + TwoSidedIntegralCompressor.MAX_VALUE + 1 + }; + try { + super.testArray(c, nonnegrange); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + public void testSupportedNonNegRange() throws IOException { + long[] nonnegrange = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + TwoSidedIntegralCompressor.MAX_VALUE + }; + super.testArray(c, nonnegrange); + } + + @Override + public void testRange() throws IOException { + try { + super.testRange(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + public void testUnsupportedRange() throws IOException { + long[] range = { + TwoSidedIntegralCompressor.MIN_VALUE - 1, + Integer.MIN_VALUE, + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + TwoSidedIntegralCompressor.MAX_VALUE + }; + try { + super.testArray(c, range); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + public void testSupportedRange() throws IOException { + long[] range = { + TwoSidedIntegralCompressor.MIN_VALUE, + Integer.MIN_VALUE, + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + TwoSidedIntegralCompressor.MAX_VALUE + }; + super.testArray(c, range); + } + + @Override + public void testNonNegAscending() throws IOException { + long[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + TwoSidedIntegralCompressor.MAX_VALUE + }; + super.testArray(c, nonnegascending); + } + + @Override + public void testAscending() throws IOException { + long[] ascending = { + TwoSidedIntegralCompressor.MIN_VALUE, + Integer.MIN_VALUE, + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + TwoSidedIntegralCompressor.MAX_VALUE + }; + super.testArray(c, ascending); + } + + @Override + public void testNonNegDescending() throws IOException { + long[] nonnegdescending = { + TwoSidedIntegralCompressor.MAX_VALUE, + Integer.MAX_VALUE, + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0 + }; + super.testArray(c, nonnegdescending); + } + + @Override + public void testDescending() throws IOException { + long[] descending = { + TwoSidedIntegralCompressor.MAX_VALUE, + Integer.MAX_VALUE, + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE, + Integer.MIN_VALUE, + TwoSidedIntegralCompressor.MIN_VALUE + }; + super.testArray(c, descending); + } + + @Override + public void testNonNegZigZag() throws IOException { + long[] nonnegzigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + TwoSidedIntegralCompressor.MAX_VALUE, + 0, + TwoSidedIntegralCompressor.MAX_VALUE + }; + super.testArray(c, nonnegzigzag); + } + + @Override + public void testZigZag() throws IOException { + long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + TwoSidedIntegralCompressor.MAX_VALUE, + TwoSidedIntegralCompressor.MIN_VALUE, + TwoSidedIntegralCompressor.MAX_VALUE + }; + super.testArray(c, zigzag); + } + } + + public static class TestEliasGammaCompressorDForBooleans extends TestBooleanCompressorBase { + public TestEliasGammaCompressorDForBooleans() { + super(new EliasGammaCompressorD()); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestEliasGammaCompressorDForBytes extends TestByteCompressorBase { + public TestEliasGammaCompressorDForBytes() { + super(new EliasGammaCompressorD()); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegZigZag() throws IOException { + try { + super.testNonNegZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + } + + public static class TestEliasGammaCompressorDForShorts extends TestShortCompressorBase { + public TestEliasGammaCompressorDForShorts() { + super(new EliasGammaCompressorD()); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegZigZag() throws IOException { + try { + super.testNonNegZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestEliasGammaCompressorDForInts extends TestIntCompressorBase { + public TestEliasGammaCompressorDForInts() { + super(new EliasGammaCompressorD()); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegZigZag() throws IOException { + try { + super.testNonNegZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestEliasGammaCompressorDForLongs extends TestLongCompressorBase { + public TestEliasGammaCompressorDForLongs() { + super(new EliasGammaCompressorD()); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegZigZag() throws IOException { + try { + super.testNonNegZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestEliasGammaCompressorD2SForBooleans extends TestBooleanCompressorBase { + public TestEliasGammaCompressorD2SForBooleans() { + super(new EliasGammaCompressorD2S()); + } + } + + public static class TestEliasGammaCompressorD2SForBytes extends TestByteCompressorBase { + public TestEliasGammaCompressorD2SForBytes() { + super(new EliasGammaCompressorD2S()); + } + } + + public static class TestEliasGammaCompressorD2SForShorts extends TestShortCompressorBase { + public TestEliasGammaCompressorD2SForShorts() { + super(new EliasGammaCompressorD2S()); + } + } + + public static class TestEliasGammaCompressorD2SForInts extends TestIntCompressorBase { + public TestEliasGammaCompressorD2SForInts() { + super(new EliasGammaCompressorD2S()); + } + } + + public static class TestEliasGammaCompressorD2SForLongs extends TestLongCompressorBase { + public TestEliasGammaCompressorD2SForLongs() { + super(new EliasGammaCompressorD2S()); + } + + @Override + public void testNonNegZigZag() throws IOException { + try { + super.testNonNegZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + public void testUnsupportedNonNegZigZag1() throws IOException { + long[] nonnegzigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + TwoSidedIntegralCompressor.MAX_VALUE + 40 + 1 // diff = 1 + max + }; + try { + super.testArray(c, nonnegzigzag); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + public void testUnsupportedNonNegZigZag2() throws IOException { + long[] nonnegzigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + TwoSidedIntegralCompressor.MAX_VALUE + 40, + // diff = min - 1 + TwoSidedIntegralCompressor.MAX_VALUE + 40 + TwoSidedIntegralCompressor.MIN_VALUE - 1 + }; + try { + super.testArray(c, nonnegzigzag); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + + public void testSupportedNonNegZigZag() throws IOException { + long[] nonnegzigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + TwoSidedIntegralCompressor.MAX_VALUE + 40, + TwoSidedIntegralCompressor.MAX_VALUE + 40 + TwoSidedIntegralCompressor.MIN_VALUE + }; + super.testArray(c, nonnegzigzag); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException ignore) { + return; + } + fail("Expected exception not thrown"); + } + } + +} diff --git contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestIntCompressorBase.java contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestIntCompressorBase.java new file mode 100644 index 0000000..eb36f5c --- /dev/null +++ contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestIntCompressorBase.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.contrib.ubercompressor.InputReader; +import org.apache.hadoop.hive.contrib.ubercompressor.OutputWriter; +import org.apache.hadoop.hive.contrib.ubercompressor.TypeSpecificCompressor; +import org.apache.hadoop.hive.contrib.ubercompressor.dsalg.Tuple; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; + +public class TestIntCompressorBase extends TestCase { + + protected TypeSpecificCompressor c; + + TestIntCompressorBase(TypeSpecificCompressor c) { + this.c = c; + } + + private int[] nonnegrange = { + 0, + Integer.MAX_VALUE + }; + + private int[] range = { + Integer.MIN_VALUE, + Integer.MAX_VALUE + }; + + private int[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE + }; + + private int[] ascending = { + Integer.MIN_VALUE, + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE + }; + + private int[] nonnegdescending = { + Integer.MAX_VALUE, + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0 + }; + + private int[] descending = { + Integer.MAX_VALUE, + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE, + Integer.MIN_VALUE + }; + + private int[] nonnegzigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Integer.MAX_VALUE, + 0, + Integer.MAX_VALUE + }; + + private int[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Integer.MAX_VALUE, + Integer.MIN_VALUE, + Integer.MAX_VALUE + }; + + protected void testArray(TypeSpecificCompressor c, int[] arr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IntObjectInspector boi = (IntObjectInspector) + PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveCategory.INT); + OutputWriter ow = c.createOutputWriter(out, boi); + for (Integer b : arr) { + ow.write(b); + } + long numBits = ow.close(true); + byte[] byteArray = out.toByteArray(); + ByteArrayInputStream in = new ByteArrayInputStream(byteArray); + InputReader ir = c.createInputReader(in, numBits); + + for (Integer exp : arr) { + Tuple i = ir.read(); + assertFalse(i.getFirst()); + assertEquals(exp, i.getSecond()); + } + Tuple i = ir.read(); + assertTrue(i.getFirst()); + } + + public void testNonNegRange() throws IOException { + testArray(c, nonnegrange); + } + + public void testRange() throws IOException { + testArray(c, range); + } + + public void testNonNegAscending() throws IOException { + testArray(c, nonnegascending); + } + + public void testAscending() throws IOException { + testArray(c, ascending); + } + + public void testDescending() throws IOException { + testArray(c, descending); + } + + public void testNonNegDescending() throws IOException { + testArray(c, nonnegdescending); + } + + public void testNonNegZigZag() throws IOException { + testArray(c, nonnegzigzag); + } + + public void testZigZag() throws IOException { + testArray(c, zigzag); + } + +} diff --git contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestLongCompressorBase.java contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestLongCompressorBase.java new file mode 100644 index 0000000..2a646e9 --- /dev/null +++ contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestLongCompressorBase.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.contrib.ubercompressor.InputReader; +import org.apache.hadoop.hive.contrib.ubercompressor.OutputWriter; +import org.apache.hadoop.hive.contrib.ubercompressor.TypeSpecificCompressor; +import org.apache.hadoop.hive.contrib.ubercompressor.dsalg.Tuple; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; + +public class TestLongCompressorBase extends TestCase { + + protected TypeSpecificCompressor c; + + TestLongCompressorBase(TypeSpecificCompressor c) { + this.c = c; + } + + private long[] nonnegrange = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + Integer.MAX_VALUE + TwoSidedIntegralCompressor.MAX_VALUE, + Long.MAX_VALUE + }; + + private long[] range = { + Long.MIN_VALUE, + Integer.MIN_VALUE + TwoSidedIntegralCompressor.MIN_VALUE, + Integer.MIN_VALUE, + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + Integer.MAX_VALUE + TwoSidedIntegralCompressor.MAX_VALUE, + Long.MAX_VALUE + }; + + private long[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + Integer.MAX_VALUE + TwoSidedIntegralCompressor.MAX_VALUE, + Long.MAX_VALUE + }; + + private long[] ascending = { + Long.MIN_VALUE, + Integer.MIN_VALUE + TwoSidedIntegralCompressor.MIN_VALUE, + Integer.MIN_VALUE, + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + Integer.MAX_VALUE, + Integer.MAX_VALUE + TwoSidedIntegralCompressor.MAX_VALUE, + Long.MAX_VALUE + }; + + private long[] nonnegdescending = { + Long.MAX_VALUE, + Integer.MAX_VALUE + TwoSidedIntegralCompressor.MAX_VALUE, + Integer.MAX_VALUE, + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0 + }; + + private long[] descending = { + Long.MAX_VALUE, + Integer.MAX_VALUE + TwoSidedIntegralCompressor.MAX_VALUE, + Integer.MAX_VALUE, + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE, + Integer.MIN_VALUE, + Integer.MIN_VALUE + TwoSidedIntegralCompressor.MIN_VALUE, + Long.MIN_VALUE + }; + + private long[] nonnegzigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Long.MAX_VALUE, + 0, + Long.MAX_VALUE + }; + + private long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Long.MAX_VALUE, + Long.MIN_VALUE, + Long.MAX_VALUE + }; + + protected void testArray(TypeSpecificCompressor c, long[] arr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + LongObjectInspector boi = (LongObjectInspector) + PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG); + OutputWriter ow = c.createOutputWriter(out, boi); + for (Long b : arr) { + ow.write(b); + } + long numBits = ow.close(true); + byte[] byteArray = out.toByteArray(); + ByteArrayInputStream in = new ByteArrayInputStream(byteArray); + InputReader ir = c.createInputReader(in, numBits); + + for (Long exp : arr) { + Tuple i = ir.read(); + assertFalse(i.getFirst()); + assertEquals(exp, i.getSecond()); + } + Tuple i = ir.read(); + assertTrue(i.getFirst()); + } + + public void testNonNegRange() throws IOException { + testArray(c, nonnegrange); + } + + public void testRange() throws IOException { + testArray(c, range); + } + + public void testNonNegAscending() throws IOException { + testArray(c, nonnegascending); + } + + public void testAscending() throws IOException { + testArray(c, ascending); + } + + public void testNonNegDescending() throws IOException { + testArray(c, nonnegdescending); + } + + public void testDescending() throws IOException { + testArray(c, descending); + } + + public void testNonNegZigZag() throws IOException { + testArray(c, nonnegzigzag); + } + + public void testZigZag() throws IOException { + testArray(c, zigzag); + } +} diff --git contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestShortCompressorBase.java contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestShortCompressorBase.java new file mode 100644 index 0000000..2962954 --- /dev/null +++ contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestShortCompressorBase.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.contrib.ubercompressor.InputReader; +import org.apache.hadoop.hive.contrib.ubercompressor.OutputWriter; +import org.apache.hadoop.hive.contrib.ubercompressor.TypeSpecificCompressor; +import org.apache.hadoop.hive.contrib.ubercompressor.dsalg.Tuple; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; + +public class TestShortCompressorBase extends TestCase { + + private TypeSpecificCompressor c; + + TestShortCompressorBase(TypeSpecificCompressor c) { + this.c = c; + } + + private short[] nonnegrange = { + 0, + Short.MAX_VALUE + }; + + private short[] range = { + Short.MIN_VALUE, + Short.MAX_VALUE + }; + + private short[] nonnegascending = { + 0, + 4, + 16, + 256, + Short.MAX_VALUE + }; + + + private short[] ascending = { + Short.MIN_VALUE, + -256, + -16, + -4, + 0, + 4, + 16, + 256, + Short.MAX_VALUE + }; + + private short[] nonnegdescending = { + Short.MAX_VALUE, + 256, + 16, + 4, + 0 + }; + + private short[] descending = { + Short.MAX_VALUE, + 256, + 16, + 4, + 0, + -4, + -16, + -256, + Short.MIN_VALUE + }; + + private short[] nonnegzigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + 0, + Short.MAX_VALUE + }; + + private short[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + Short.MIN_VALUE, + Short.MAX_VALUE + }; + + protected void testArray(TypeSpecificCompressor c, short[] arr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ShortObjectInspector boi = (ShortObjectInspector) + PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveCategory.SHORT); + OutputWriter ow = c.createOutputWriter(out, boi); + for (Short b : arr) { + ow.write(b); + } + long numBits = ow.close(true); + byte[] byteArray = out.toByteArray(); + ByteArrayInputStream in = new ByteArrayInputStream(byteArray); + InputReader ir = c.createInputReader(in, numBits); + + for (Short exp : arr) { + Tuple i = ir.read(); + assertFalse(i.getFirst()); + assertEquals(exp, i.getSecond()); + } + Tuple i = ir.read(); + assertTrue(i.getFirst()); + } + + public void testNonNegRange() throws IOException { + testArray(c, nonnegrange); + } + + public void testRange() throws IOException { + testArray(c, range); + } + + public void testNonNegAscending() throws IOException { + testArray(c, nonnegascending); + } + + public void testAscending() throws IOException { + testArray(c, ascending); + } + + public void testNonNegDescending() throws IOException { + testArray(c, nonnegdescending); + } + + public void testDescending() throws IOException { + testArray(c, descending); + } + + public void testNonNegZigZag() throws IOException { + testArray(c, nonnegzigzag); + } + + public void testZigZag() throws IOException { + testArray(c, zigzag); + } + +} diff --git contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestUnaryCompressor.java contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestUnaryCompressor.java new file mode 100644 index 0000000..8ce1c16 --- /dev/null +++ contrib/src/test/org/apache/hadoop/hive/contrib/ubercompressor/compressors/TestUnaryCompressor.java @@ -0,0 +1,1170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.contrib.ubercompressor.compressors; + +import java.io.IOException; + +import junit.framework.Test; +import junit.framework.TestSuite; + +public class TestUnaryCompressor { + public static Test suite() { + TestSuite suite = new TestSuite(TestUnaryCompressor.class.getName()); + suite.addTestSuite(TestUnaryCompressorForBooleans.class); + suite.addTestSuite(TestUnaryCompressorForBytes.class); + suite.addTestSuite(TestUnaryCompressorForShorts.class); + suite.addTestSuite(TestUnaryCompressorForInts.class); + suite.addTestSuite(TestUnaryCompressorForLongs.class); + + suite.addTestSuite(TestUnaryCompressor2SForBooleans.class); + suite.addTestSuite(TestUnaryCompressor2SForBytes.class); + suite.addTestSuite(TestUnaryCompressor2SForShorts.class); + suite.addTestSuite(TestUnaryCompressor2SForInts.class); + suite.addTestSuite(TestUnaryCompressor2SForLongs.class); + + suite.addTestSuite(TestUnaryCompressorDForBooleans.class); + suite.addTestSuite(TestUnaryCompressorDForBytes.class); + suite.addTestSuite(TestUnaryCompressorDForShorts.class); + suite.addTestSuite(TestUnaryCompressorDForInts.class); + suite.addTestSuite(TestUnaryCompressorDForLongs.class); + + suite.addTestSuite(TestUnaryCompressorD2SForBooleans.class); + suite.addTestSuite(TestUnaryCompressorD2SForBytes.class); + suite.addTestSuite(TestUnaryCompressorD2SForShorts.class); + suite.addTestSuite(TestUnaryCompressorD2SForInts.class); + suite.addTestSuite(TestUnaryCompressorD2SForLongs.class); + + return suite; + } + + public static class TestUnaryCompressorForBooleans extends TestBooleanCompressorBase { + public TestUnaryCompressorForBooleans() { + super(new UnaryCompressor()); + } + } + + public static class TestUnaryCompressorForBytes extends TestByteCompressorBase { + + public TestUnaryCompressorForBytes() { + super(new UnaryCompressor()); + } + + @Override + public void testRange() throws IOException { + try { + super.testRange(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testAscending() throws IOException { + try { + super.testAscending(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testDescending() throws IOException { + try { + super.testDescending(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestUnaryCompressorForShorts extends TestShortCompressorBase { + + public TestUnaryCompressorForShorts() { + super(new UnaryCompressor()); + } + + @Override + public void testRange() throws IOException { + try { + super.testRange(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testAscending() throws IOException { + try { + super.testAscending(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testDescending() throws IOException { + try { + super.testDescending(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestUnaryCompressorForInts extends TestIntCompressorBase { + + public TestUnaryCompressorForInts() { + super(new UnaryCompressor()); + } + + @Override + public void testNonNegRange() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] nonnegrange = { + 0, + Short.MAX_VALUE + }; + super.testArray(c, nonnegrange); + } + + @Override + public void testRange() throws IOException { + try { + super.testRange(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegAscending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, nonnegascending); + } + + @Override + public void testAscending() throws IOException { + try { + super.testAscending(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + }; + super.testArray(c, descending); + } + + @Override + public void testDescending() throws IOException { + try { + /* the base class test is grossly inefficient for unary coding */ + int[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE + }; + super.testArray(c, descending); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegZigZag() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + 0, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + + @Override + public void testZigZag() throws IOException { + try { + /* the base class test is grossly inefficient for unary coding */ + int[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestUnaryCompressorForLongs extends TestLongCompressorBase { + + public TestUnaryCompressorForLongs() { + super(new UnaryCompressor()); + } + + @Override + public void testNonNegRange() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] nonnegrange = { + 0, + Short.MAX_VALUE + }; + super.testArray(c, nonnegrange); + } + + @Override + public void testRange() throws IOException { + try { + super.testRange(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegAscending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, nonnegascending); + } + + @Override + public void testAscending() throws IOException { + try { + super.testAscending(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + }; + super.testArray(c, descending); + } + + @Override + public void testDescending() throws IOException { + try { + /* the base class test is grossly inefficient for unary coding */ + long[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE + }; + super.testArray(c, descending); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testNonNegZigZag() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + 0, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + + @Override + public void testZigZag() throws IOException { + try { + /* the base class test is grossly inefficient for unary coding */ + long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestUnaryCompressor2SForBooleans extends TestBooleanCompressorBase { + public TestUnaryCompressor2SForBooleans() { + super(new UnaryCompressor2S()); + } + } + + public static class TestUnaryCompressor2SForBytes extends TestByteCompressorBase { + + public TestUnaryCompressor2SForBytes() { + super(new UnaryCompressor2S()); + } + } + + public static class TestUnaryCompressor2SForShorts extends TestShortCompressorBase { + + public TestUnaryCompressor2SForShorts() { + super(new UnaryCompressor2S()); + } + } + + public static class TestUnaryCompressor2SForInts extends TestIntCompressorBase { + + public TestUnaryCompressor2SForInts() { + super(new UnaryCompressor2S()); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testNonNegRange() throws IOException { + int[] nonnegrange = { + 0, + Short.MAX_VALUE + }; + super.testArray(c, nonnegrange); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testRange() throws IOException { + int[] range = { + Short.MIN_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, range); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testNonNegAscending() throws IOException { + int[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, nonnegascending); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testAscending() throws IOException { + int[] ascending = { + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, ascending); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testNonNegDescending() throws IOException { + int[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + }; + super.testArray(c, descending); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testDescending() throws IOException { + int[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE + }; + super.testArray(c, descending); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testNonNegZigZag() throws IOException { + int[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + 0, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testZigZag() throws IOException { + int[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + } + + public static class TestUnaryCompressor2SForLongs extends TestLongCompressorBase { + + + public TestUnaryCompressor2SForLongs() { + super(new UnaryCompressor2S()); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testNonNegRange() throws IOException { + long[] nonnegrange = { + 0, + Short.MAX_VALUE + }; + super.testArray(c, nonnegrange); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testRange() throws IOException { + long[] range = { + Short.MIN_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, range); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testNonNegAscending() throws IOException { + long[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, nonnegascending); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testAscending() throws IOException { + long[] ascending = { + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, ascending); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testNonNegDescending() throws IOException { + long[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + }; + super.testArray(c, descending); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testDescending() throws IOException { + long[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE + }; + super.testArray(c, descending); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testNonNegZigZag() throws IOException { + long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + 0, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + + /* the base class test is grossly inefficient for unary coding */ + public void testZigZag() throws IOException { + long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + } + + public static class TestUnaryCompressorDForBooleans extends TestBooleanCompressorBase { + public TestUnaryCompressorDForBooleans() { + super(new UnaryCompressorD()); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestUnaryCompressorDForBytes extends TestByteCompressorBase { + public TestUnaryCompressorDForBytes() { + super(new UnaryCompressorD()); + } + + @Override + public void testNonNegZigZag() throws IOException { + try { + super.testNonNegZigZag(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + } + + + public static class TestUnaryCompressorDForShorts extends TestShortCompressorBase { + public TestUnaryCompressorDForShorts() { + super(new UnaryCompressorD()); + } + + @Override + public void testNonNegZigZag() throws IOException { + try { + super.testNonNegZigZag(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + super.testZigZag(); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + } + + public static class TestUnaryCompressorDForInts extends TestIntCompressorBase { + public TestUnaryCompressorDForInts() { + super(new UnaryCompressorD()); + } + + @Override + public void testNonNegRange() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] nonnegrange = { + 0, + Short.MAX_VALUE + }; + super.testArray(c, nonnegrange); + } + + @Override + public void testRange() throws IOException { + int[] range = { + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, range); + } + + @Override + public void testNonNegAscending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, nonnegascending); + } + + @Override + public void testAscending() throws IOException { + int[] ascending = { + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, ascending); + } + + @Override + public void testNonNegDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + }; + super.testArray(c, descending); + } + + @Override + public void testDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE + }; + super.testArray(c, descending); + } + + @Override + public void testNonNegZigZag() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + 0, + Short.MAX_VALUE + }; + try { + super.testArray(c, zigzag); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + /* the base class test is grossly inefficient for unary coding */ + int[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + } + + public static class TestUnaryCompressorDForLongs extends TestLongCompressorBase { + public TestUnaryCompressorDForLongs() { + super(new UnaryCompressorD()); + } + + @Override + public void testNonNegRange() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] nonnegrange = { + 0, + Short.MAX_VALUE + }; + super.testArray(c, nonnegrange); + } + + @Override + public void testRange() throws IOException { + long[] range = { + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, range); + } + + @Override + public void testNonNegAscending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, nonnegascending); + } + + @Override + public void testAscending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] ascending = { + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, ascending); + } + + @Override + public void testNonNegDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + }; + super.testArray(c, descending); + } + + @Override + public void testDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE + }; + super.testArray(c, descending); + } + + @Override + public void testNonNegZigZag() throws IOException { + try { + /* the base class test is grossly inefficient for unary coding */ + long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + 0, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + @Override + public void testZigZag() throws IOException { + try { + /* the base class test is grossly inefficient for unary coding */ + long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } catch (IllegalArgumentException expected) { + return; + } + fail("Expected exception not thrown"); + } + + } + + public static class TestUnaryCompressorD2SForBooleans extends TestBooleanCompressorBase { + public TestUnaryCompressorD2SForBooleans() { + super(new UnaryCompressorD2S()); + } + } + + public static class TestUnaryCompressorD2SForBytes extends TestByteCompressorBase { + public TestUnaryCompressorD2SForBytes() { + super(new UnaryCompressorD2S()); + } + } + + public static class TestUnaryCompressorD2SForShorts extends TestShortCompressorBase { + public TestUnaryCompressorD2SForShorts() { + super(new UnaryCompressorD2S()); + } + } + + public static class TestUnaryCompressorD2SForInts extends TestIntCompressorBase { + public TestUnaryCompressorD2SForInts() { + super(new UnaryCompressorD2S()); + } + + @Override + public void testNonNegRange() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] nonnegrange = { + 0, + Short.MAX_VALUE + }; + super.testArray(c, nonnegrange); + } + + @Override + public void testRange() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] range = { + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, range); + } + + @Override + public void testNonNegAscending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, nonnegascending); + } + + @Override + public void testAscending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] ascending = { + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, ascending); + } + + @Override + public void testNonNegDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + }; + super.testArray(c, descending); + } + + @Override + public void testDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE + }; + super.testArray(c, descending); + } + + @Override + public void testNonNegZigZag() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + 0, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + + @Override + public void testZigZag() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + int[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + } + + public static class TestUnaryCompressorD2SForLongs extends TestLongCompressorBase { + public TestUnaryCompressorD2SForLongs() { + super(new UnaryCompressorD2S()); + } + + @Override + public void testNonNegRange() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] nonnegrange = { + 0, + Short.MAX_VALUE + }; + super.testArray(c, nonnegrange); + } + + @Override + public void testRange() throws IOException { + long[] range = { + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, range); + } + + @Override + public void testNonNegAscending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] nonnegascending = { + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, nonnegascending); + } + + @Override + public void testAscending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] ascending = { + Short.MIN_VALUE, + Byte.MIN_VALUE, + 0, + Byte.MAX_VALUE, + Short.MAX_VALUE, + }; + super.testArray(c, ascending); + } + + @Override + public void testNonNegDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + }; + super.testArray(c, descending); + } + + @Override + public void testDescending() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] descending = { + Short.MAX_VALUE, + Byte.MAX_VALUE, + 0, + Byte.MIN_VALUE, + Short.MIN_VALUE + }; + super.testArray(c, descending); + } + + @Override + public void testNonNegZigZag() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + 0, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + + @Override + public void testZigZag() throws IOException { + /* the base class test is grossly inefficient for unary coding */ + long[] zigzag = { + 50, + 45, + 40, + 45, + 50, + 45, + 40, + Short.MAX_VALUE, + Short.MIN_VALUE, + Short.MAX_VALUE + }; + super.testArray(c, zigzag); + } + + } + +}