Index: build.xml =================================================================== --- build.xml (revision 1440302) +++ build.xml (working copy) @@ -684,6 +684,7 @@ + Index: contrib/ivy.xml =================================================================== --- contrib/ivy.xml (revision 1440302) +++ contrib/ivy.xml (working copy) @@ -36,7 +36,19 @@ transitive="false"/> + + + + + + + Index: contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (revision 0) +++ contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (working copy) @@ -0,0 +1,777 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import static junit.framework.Assert.*; + +/** + * Tests for the top level reader/streamFactory of ORC files. + */ +public class TestOrcFile { + + public static class InnerStruct { + int int1; + Text string1 = new Text(); + InnerStruct(int int1, String string1) { + this.int1 = int1; + this.string1.set(string1); + } + } + + public static class MiddleStruct { + List list = new ArrayList(); + + MiddleStruct(InnerStruct... items) { + list.clear(); + for(InnerStruct item: items) { + list.add(item); + } + } + } + + public static class BigRow { + Boolean boolean1; + Byte byte1; + Short short1; + Integer int1; + Long long1; + Float float1; + Double double1; + BytesWritable bytes1; + Text string1; + MiddleStruct middle; + List list = new ArrayList(); + Map map = new HashMap(); + + BigRow(Boolean b1, Byte b2, Short s1, Integer i1, Long l1, Float f1, + Double d1, + BytesWritable b3, String s2, MiddleStruct m1, + List l2, Map m2) { + this.boolean1 = b1; + this.byte1 = b2; + this.short1 = s1; + this.int1 = i1; + this.long1 = l1; + this.float1 = f1; + this.double1 = d1; + this.bytes1 = b3; + if (s2 == null) { + this.string1 = null; + } else { + this.string1 = new Text(s2); + } + this.middle = m1; + this.list = l2; + this.map = m2; + } + } + + private static InnerStruct inner(int i, String s) { + return new InnerStruct(i, s); + } + + private static Map map(InnerStruct... items) { + Map result = new HashMap(); + for(InnerStruct i: items) { + result.put(new Text(i.string1), i); + } + return result; + } + + private static List list(InnerStruct... items) { + List result = new ArrayList(); + for(InnerStruct s: items) { + result.add(s); + } + return result; + } + + private static BytesWritable bytes(int... items) { + BytesWritable result = new BytesWritable(); + result.setSize(items.length); + for(int i=0; i < items.length; ++i) { + result.getBytes()[i] = (byte) items[i]; + } + return result; + } + + private static ByteBuffer byteBuf(int... items) { + ByteBuffer result = ByteBuffer.allocate(items.length); + for(int item: items) { + result.put((byte) item); + } + return result; + } + + Path workDir = new Path(System.getProperty("test.tmp.dir", + "target/test/tmp")); + + @Test + public void test1() throws Exception { + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + ObjectInspector inspector = + ObjectInspectorFactory.getReflectionObjectInspector(BigRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + Writer writer = OrcFile.createWriter(fs, p, inspector, + 100000, CompressionKind.ZLIB, 10000, 10000); + writer.addRow(new BigRow(false, (byte) 1, (short) 1024, 65536, + Long.MAX_VALUE, (float) 1.0, -15.0, bytes(0,1,2,3,4), "hi", + new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), + list(inner(3, "good"), inner(4, "bad")), + map())); + writer.addRow(new BigRow(true, (byte) 100, (short) 2048, 65536, + Long.MAX_VALUE, (float) 2.0, -5.0, bytes(), "bye", + new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), + list(inner(100000000, "cat"), inner(-100000, "in"), inner(1234, "hat")), + map(inner(5,"chani"), inner(1,"mauddib")))); + writer.close(); + Reader reader = OrcFile.createReader(fs, p); + + // check the stats + ColumnStatistics[] stats = reader.getStatistics(); + assertEquals(2, stats[1].getNumberOfValues()); + assertEquals(1, ((BooleanColumnStatistics) stats[1]).getFalseCount()); + assertEquals(1, ((BooleanColumnStatistics) stats[1]).getTrueCount()); + assertEquals("count: 2 true: 1", stats[1].toString()); + + assertEquals(2048, ((IntegerColumnStatistics) stats[3]).getMaximum()); + assertEquals(1024, ((IntegerColumnStatistics) stats[3]).getMinimum()); + assertEquals(true, ((IntegerColumnStatistics) stats[3]).isSumDefined()); + assertEquals(3072, ((IntegerColumnStatistics) stats[3]).getSum()); + assertEquals("count: 2 min: 1024 max: 2048 sum: 3072", + stats[3].toString()); + + assertEquals(Long.MAX_VALUE, + ((IntegerColumnStatistics) stats[5]).getMaximum()); + assertEquals(Long.MAX_VALUE, + ((IntegerColumnStatistics) stats[5]).getMinimum()); + assertEquals(false, ((IntegerColumnStatistics) stats[5]).isSumDefined()); + assertEquals("count: 2 min: 9223372036854775807 max: 9223372036854775807", + stats[5].toString()); + + assertEquals(-15.0, ((DoubleColumnStatistics) stats[7]).getMinimum()); + assertEquals(-5.0, ((DoubleColumnStatistics) stats[7]).getMaximum()); + assertEquals(-20.0, ((DoubleColumnStatistics) stats[7]).getSum(), 0.00001); + assertEquals("count: 2 min: -15.0 max: -5.0 sum: -20.0", + stats[7].toString()); + + assertEquals("count: 2 min: bye max: hi", stats[9].toString()); + + // check the inspectors + StructObjectInspector readerInspector = + (StructObjectInspector) reader.getObjectInspector(); + assertEquals(ObjectInspector.Category.STRUCT, + readerInspector.getCategory()); + assertEquals("struct>>,list:list>," + + "map:map>>", + readerInspector.getTypeName()); + List fields = + readerInspector.getAllStructFieldRefs(); + BooleanObjectInspector bo = (BooleanObjectInspector) readerInspector. + getStructFieldRef("boolean1").getFieldObjectInspector(); + ByteObjectInspector by = (ByteObjectInspector) readerInspector. + getStructFieldRef("byte1").getFieldObjectInspector(); + ShortObjectInspector sh = (ShortObjectInspector) readerInspector. + getStructFieldRef("short1").getFieldObjectInspector(); + IntObjectInspector in = (IntObjectInspector) readerInspector. + getStructFieldRef("int1").getFieldObjectInspector(); + LongObjectInspector lo = (LongObjectInspector) readerInspector. + getStructFieldRef("long1").getFieldObjectInspector(); + FloatObjectInspector fl = (FloatObjectInspector) readerInspector. + getStructFieldRef("float1").getFieldObjectInspector(); + DoubleObjectInspector dbl = (DoubleObjectInspector) readerInspector. + getStructFieldRef("double1").getFieldObjectInspector(); + BinaryObjectInspector bi = (BinaryObjectInspector) readerInspector. + getStructFieldRef("bytes1").getFieldObjectInspector(); + StringObjectInspector st = (StringObjectInspector) readerInspector. + getStructFieldRef("string1").getFieldObjectInspector(); + StructObjectInspector mid = (StructObjectInspector) readerInspector. + getStructFieldRef("middle").getFieldObjectInspector(); + List midFields = + mid.getAllStructFieldRefs(); + ListObjectInspector midli = + (ListObjectInspector) midFields.get(0).getFieldObjectInspector(); + StructObjectInspector inner = (StructObjectInspector) + midli.getListElementObjectInspector(); + List inFields = inner.getAllStructFieldRefs(); + ListObjectInspector li = (ListObjectInspector) readerInspector. + getStructFieldRef("list").getFieldObjectInspector(); + MapObjectInspector ma = (MapObjectInspector) readerInspector. + getStructFieldRef("map").getFieldObjectInspector(); + StructObjectInspector lc = (StructObjectInspector) + li.getListElementObjectInspector(); + StringObjectInspector mk = (StringObjectInspector) + ma.getMapKeyObjectInspector(); + StructObjectInspector mv = (StructObjectInspector) + ma.getMapValueObjectInspector(); + RecordReader rows = reader.rows(null); + Object row = rows.next(null); + assertNotNull(row); + // check the contents of the first row + assertEquals(false, + bo.get(readerInspector.getStructFieldData(row, fields.get(0)))); + assertEquals(1, by.get(readerInspector.getStructFieldData(row, + fields.get(1)))); + assertEquals(1024, sh.get(readerInspector.getStructFieldData(row, + fields.get(2)))); + assertEquals(65536, in.get(readerInspector.getStructFieldData(row, + fields.get(3)))); + assertEquals(Long.MAX_VALUE, lo.get(readerInspector. + getStructFieldData(row, fields.get(4)))); + assertEquals(1.0, fl.get(readerInspector.getStructFieldData(row, + fields.get(5))), 0.00001); + assertEquals(-15.0, dbl.get(readerInspector.getStructFieldData(row, + fields.get(6))), 0.00001); + assertEquals(bytes(0,1,2,3,4), bi.getPrimitiveWritableObject( + readerInspector.getStructFieldData(row, fields.get(7)))); + assertEquals("hi", st.getPrimitiveJavaObject(readerInspector. + getStructFieldData(row, fields.get(8)))); + List midRow = midli.getList(mid.getStructFieldData(readerInspector. + getStructFieldData(row, fields.get(9)), midFields.get(0))); + assertNotNull(midRow); + assertEquals(2, midRow.size()); + assertEquals(1, in.get(inner.getStructFieldData(midRow.get(0), + inFields.get(0)))); + assertEquals("bye", st.getPrimitiveJavaObject(inner.getStructFieldData + (midRow.get(0), inFields.get(1)))); + assertEquals(2, in.get(inner.getStructFieldData(midRow.get(1), + inFields.get(0)))); + assertEquals("sigh", st.getPrimitiveJavaObject(inner.getStructFieldData + (midRow.get(1), inFields.get(1)))); + List list = li.getList(readerInspector.getStructFieldData(row, + fields.get(10))); + assertEquals(2, list.size()); + assertEquals(3, in.get(inner.getStructFieldData(list.get(0), + inFields.get(0)))); + assertEquals("good", st.getPrimitiveJavaObject(inner.getStructFieldData + (list.get(0), inFields.get(1)))); + assertEquals(4, in.get(inner.getStructFieldData(list.get(1), + inFields.get(0)))); + assertEquals("bad", st.getPrimitiveJavaObject(inner.getStructFieldData + (list.get(1), inFields.get(1)))); + Map map = ma.getMap(readerInspector.getStructFieldData(row, + fields.get(11))); + assertEquals(0, map.size()); + + // check the contents of second row + assertEquals(true, rows.hasNext()); + row = rows.next(row); + assertEquals(true, + bo.get(readerInspector.getStructFieldData(row, fields.get(0)))); + assertEquals(100, by.get(readerInspector.getStructFieldData(row, + fields.get(1)))); + assertEquals(2048, sh.get(readerInspector.getStructFieldData(row, + fields.get(2)))); + assertEquals(65536, in.get(readerInspector.getStructFieldData(row, + fields.get(3)))); + assertEquals(Long.MAX_VALUE, lo.get(readerInspector. + getStructFieldData(row, fields.get(4)))); + assertEquals(2.0, fl.get(readerInspector.getStructFieldData(row, + fields.get(5))), 0.00001); + assertEquals(-5.0, dbl.get(readerInspector.getStructFieldData(row, + fields.get(6))), 0.00001); + assertEquals(bytes(), bi.getPrimitiveWritableObject( + readerInspector.getStructFieldData(row, fields.get(7)))); + assertEquals("bye", st.getPrimitiveJavaObject(readerInspector. + getStructFieldData(row, fields.get(8)))); + midRow = midli.getList(mid.getStructFieldData(readerInspector. + getStructFieldData(row, fields.get(9)), midFields.get(0))); + assertNotNull(midRow); + assertEquals(2, midRow.size()); + assertEquals(1, in.get(inner.getStructFieldData(midRow.get(0), + inFields.get(0)))); + assertEquals("bye", st.getPrimitiveJavaObject(inner.getStructFieldData + (midRow.get(0), inFields.get(1)))); + assertEquals(2, in.get(inner.getStructFieldData(midRow.get(1), + inFields.get(0)))); + assertEquals("sigh", st.getPrimitiveJavaObject(inner.getStructFieldData + (midRow.get(1), inFields.get(1)))); + list = li.getList(readerInspector.getStructFieldData(row, + fields.get(10))); + assertEquals(3, list.size()); + assertEquals(100000000, in.get(inner.getStructFieldData(list.get(0), + inFields.get(0)))); + assertEquals("cat", st.getPrimitiveJavaObject(inner.getStructFieldData + (list.get(0), inFields.get(1)))); + assertEquals(-100000, in.get(inner.getStructFieldData(list.get(1), + inFields.get(0)))); + assertEquals("in", st.getPrimitiveJavaObject(inner.getStructFieldData + (list.get(1), inFields.get(1)))); + assertEquals(1234, in.get(inner.getStructFieldData(list.get(2), + inFields.get(0)))); + assertEquals("hat", st.getPrimitiveJavaObject(inner.getStructFieldData + (list.get(2), inFields.get(1)))); + map = ma.getMap(readerInspector.getStructFieldData(row, + fields.get(11))); + assertEquals(2, map.size()); + boolean[] found = new boolean[2]; + for(Object key: map.keySet()) { + String str = mk.getPrimitiveJavaObject(key); + if (str.equals("chani")) { + assertEquals(false, found[0]); + assertEquals(5, in.get(inner.getStructFieldData(map.get(key), + inFields.get(0)))); + assertEquals(str, st.getPrimitiveJavaObject( + inner.getStructFieldData(map.get(key), inFields.get(1)))); + found[0] = true; + } else if (str.equals("mauddib")) { + assertEquals(false, found[1]); + assertEquals(1, in.get(inner.getStructFieldData(map.get(key), + inFields.get(0)))); + assertEquals(str, st.getPrimitiveJavaObject( + inner.getStructFieldData(map.get(key), inFields.get(1)))); + found[1] = true; + } else { + throw new IllegalArgumentException("Unknown key " + str); + } + } + assertEquals(true, found[0]); + assertEquals(true, found[1]); + + // handle the close up + assertEquals(false, rows.hasNext()); + rows.close(); + } + + @Test + public void columnProjection() throws Exception { + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + ObjectInspector inspector = + ObjectInspectorFactory.getReflectionObjectInspector(InnerStruct.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Writer writer = OrcFile.createWriter(fs, p, inspector, + 1000, CompressionKind.NONE, 100, 1000); + Random r1 = new Random(1); + Random r2 = new Random(2); + int x; + int minInt=0, maxInt=0; + String y; + String minStr = null, maxStr = null; + for(int i=0; i < 21000; ++i) { + x = r1.nextInt(); + y = Long.toHexString(r2.nextLong()); + if (i == 0 || x < minInt) { + minInt = x; + } + if (i == 0 || x > maxInt) { + maxInt = x; + } + if (i == 0 || y.compareTo(minStr) < 0) { + minStr = y; + } + if (i == 0 || y.compareTo(maxStr) > 0) { + maxStr = y; + } + writer.addRow(inner(x, y)); + } + writer.close(); + Reader reader = OrcFile.createReader(fs, p); + + // check out the statistics + ColumnStatistics[] stats = reader.getStatistics(); + assertEquals(3, stats.length); + for(ColumnStatistics s: stats) { + assertEquals(21000, s.getNumberOfValues()); + if (s instanceof IntegerColumnStatistics) { + assertEquals(minInt, ((IntegerColumnStatistics) s).getMinimum()); + assertEquals(maxInt, ((IntegerColumnStatistics) s).getMaximum()); + } else if (s instanceof StringColumnStatistics) { + assertEquals(maxStr, ((StringColumnStatistics) s).getMaximum()); + assertEquals(minStr, ((StringColumnStatistics) s).getMinimum()); + } + } + + // check out the types + List types = reader.getTypes(); + assertEquals(3, types.size()); + assertEquals(OrcProto.Type.Kind.STRUCT, types.get(0).getKind()); + assertEquals(2, types.get(0).getSubtypesCount()); + assertEquals(1, types.get(0).getSubtypes(0)); + assertEquals(2, types.get(0).getSubtypes(1)); + assertEquals(OrcProto.Type.Kind.INT, types.get(1).getKind()); + assertEquals(0, types.get(1).getSubtypesCount()); + assertEquals(OrcProto.Type.Kind.STRING, types.get(2).getKind()); + assertEquals(0, types.get(2).getSubtypesCount()); + + // read the contents and make sure they match + RecordReader rows1 = reader.rows(new boolean[]{true, true, false}); + RecordReader rows2 = reader.rows(new boolean[]{true, false, true}); + r1 = new Random(1); + r2 = new Random(2); + OrcStruct row1 = null; + OrcStruct row2 = null; + for(int i = 0; i < 21000; ++i) { + assertEquals(true, rows1.hasNext()); + assertEquals(true, rows2.hasNext()); + row1 = (OrcStruct) rows1.next(row1); + row2 = (OrcStruct) rows2.next(row2); + assertEquals(r1.nextInt(), ((IntWritable) row1.getFieldValue(0)).get()); + assertEquals(Long.toHexString(r2.nextLong()), + row2.getFieldValue(1).toString()); + } + assertEquals(false, rows1.hasNext()); + assertEquals(false, rows2.hasNext()); + rows1.close(); + rows2.close(); + } + + @Test + public void emptyFile() throws Exception { + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + ObjectInspector inspector = + ObjectInspectorFactory.getReflectionObjectInspector(BigRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Writer writer = OrcFile.createWriter(fs, p, inspector, + 1000, CompressionKind.NONE, 100, 10000); + writer.close(); + Reader reader = OrcFile.createReader(fs, p); + assertEquals(false, reader.rows(null).hasNext()); + assertEquals(CompressionKind.NONE, reader.getCompression()); + assertEquals(0, reader.getNumberOfRows()); + assertEquals(0, reader.getCompressionSize()); + assertEquals(false, reader.getMetadataKeys().iterator().hasNext()); + assertEquals(3, reader.getContentLength()); + assertEquals(false, reader.getStripes().iterator().hasNext()); + } + + @Test + public void metaData() throws Exception { + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + ObjectInspector inspector = + ObjectInspectorFactory.getReflectionObjectInspector(BigRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Writer writer = OrcFile.createWriter(fs, p, inspector, + 1000, CompressionKind.NONE, 100, 10000); + writer.addUserMetadata("my.meta", byteBuf(1, 2, 3, 4, 5, 6, 7, -1, -2, 127, -128)); + writer.addUserMetadata("clobber", byteBuf(1,2,3)); + writer.addUserMetadata("clobber", byteBuf(4,3,2,1)); + ByteBuffer bigBuf = ByteBuffer.allocate(40000); + Random random = new Random(0); + random.nextBytes(bigBuf.array()); + writer.addUserMetadata("big", bigBuf); + bigBuf.position(0); + writer.addRow(new BigRow(true, (byte) 127, (short) 1024, 42, + 42L * 1024 * 1024 * 1024, (float) 3.1415, -2.713, null, + null, null, null, null)); + writer.addUserMetadata("clobber", byteBuf(5,7,11,13,17,19)); + writer.close(); + Reader reader = OrcFile.createReader(fs, p); + assertEquals(byteBuf(5,7,11,13,17,19), reader.getMetadataValue("clobber")); + assertEquals(byteBuf(1,2,3,4,5,6,7,-1,-2,127,-128), + reader.getMetadataValue("my.meta")); + assertEquals(bigBuf, reader.getMetadataValue("big")); + try { + reader.getMetadataValue("unknown"); + assertTrue(false); + } catch (IllegalArgumentException iae) { + // PASS + } + int i = 0; + for(String key: reader.getMetadataKeys()) { + if ("my.meta".equals(key) || + "clobber".equals(key) || + "big".equals(key)) { + i += 1; + } else { + throw new IllegalArgumentException("unknown key " + key); + } + } + assertEquals(3, i); + } + + /** + * We test union and timestamp separately since we need to make the + * object inspector manually. (The Hive reflection-based doesn't handle + * them properly.) + */ + @Test + public void testUnionAndTimestamp() throws Exception { + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + List types = new ArrayList(); + types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.STRUCT). + addFieldNames("time").addFieldNames("union"). + addSubtypes(1).addSubtypes(2).build()); + types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.TIMESTAMP). + build()); + types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.UNION). + addSubtypes(3).addSubtypes(4).build()); + types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.INT). + build()); + types.add(OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.STRING). + build()); + ObjectInspector inspector = OrcStruct.createObjectInspector(0, types); + Writer writer = OrcFile.createWriter(fs, p, inspector, + 1000, CompressionKind.NONE, 100, 10000); + OrcStruct row = new OrcStruct(2); + OrcUnion union = new OrcUnion(); + row.setFieldValue(1, union); + row.setFieldValue(0, Timestamp.valueOf("2000-03-12 15:00:00")); + union.set((byte) 0, new IntWritable(42)); + writer.addRow(row); + row.setFieldValue(0, Timestamp.valueOf("2000-03-20 12:00:00.123456789")); + union.set((byte)1, new Text("hello")); + writer.addRow(row); + row.setFieldValue(0, null); + row.setFieldValue(1, null); + writer.addRow(row); + row.setFieldValue(1, union); + union.set((byte) 0, null); + writer.addRow(row); + union.set((byte) 1, null); + writer.addRow(row); + union.set((byte) 0, new IntWritable(200000)); + row.setFieldValue(0, Timestamp.valueOf("1900-01-01 00:00:00")); + writer.addRow(row); + for(int i=1900; i < 2200; ++i) { + row.setFieldValue(0, Timestamp.valueOf(i + "-05-05 12:34:56." + i)); + if ((i & 1) == 0) { + union.set((byte) 0, new IntWritable(i*i)); + } else { + union.set((byte) 1, new Text(new Integer(i*i).toString())); + } + writer.addRow(row); + } + // let's add a lot of constant rows to test the rle + row.setFieldValue(0, null); + union.set((byte) 0, new IntWritable(1732050807)); + for(int i=0; i < 1000; ++i) { + writer.addRow(row); + } + union.set((byte) 0, new IntWritable(0)); + writer.addRow(row); + union.set((byte) 0, new IntWritable(10)); + writer.addRow(row); + union.set((byte) 0, new IntWritable(138)); + writer.addRow(row); + writer.close(); + Reader reader = OrcFile.createReader(fs, p); + assertEquals(false, reader.getMetadataKeys().iterator().hasNext()); + assertEquals(1309, reader.getNumberOfRows()); + int stripeCount = 0; + int rowCount = 0; + long currentOffset = -1; + for(StripeInformation stripe: reader.getStripes()) { + stripeCount += 1; + rowCount += stripe.getNumberOfRows(); + if (currentOffset < 0) { + currentOffset = stripe.getOffset() + stripe.getIndexLength() + + stripe.getDataLength() + stripe.getFooterLength(); + } else { + assertEquals(currentOffset, stripe.getOffset()); + currentOffset += stripe.getIndexLength() + + stripe.getDataLength() + stripe.getFooterLength(); + } + } + assertEquals(reader.getNumberOfRows(), rowCount); + assertEquals(2, stripeCount); + assertEquals(reader.getContentLength(), currentOffset); + RecordReader rows = reader.rows(null); + assertEquals(0, rows.getRowNumber()); + assertEquals(0.0, rows.getProgress(), 0.000001); + assertEquals(true, rows.hasNext()); + row = (OrcStruct) rows.next(null); + inspector = reader.getObjectInspector(); + assertEquals("struct", + inspector.getTypeName()); + assertEquals(Timestamp.valueOf("2000-03-12 15:00:00"), + row.getFieldValue(0)); + union = (OrcUnion) row.getFieldValue(1); + assertEquals(0, union.getTag()); + assertEquals(new IntWritable(42), union.getObject()); + row = (OrcStruct) rows.next(row); + assertEquals(Timestamp.valueOf("2000-03-20 12:00:00.123456789"), + row.getFieldValue(0)); + assertEquals(1, union.getTag()); + assertEquals(new Text("hello"), union.getObject()); + row = (OrcStruct) rows.next(row); + assertEquals(null, row.getFieldValue(0)); + assertEquals(null, row.getFieldValue(1)); + row = (OrcStruct) rows.next(row); + assertEquals(null, row.getFieldValue(0)); + union = (OrcUnion) row.getFieldValue(1); + assertEquals(0, union.getTag()); + assertEquals(null, union.getObject()); + row = (OrcStruct) rows.next(row); + assertEquals(null, row.getFieldValue(0)); + assertEquals(1, union.getTag()); + assertEquals(null, union.getObject()); + row = (OrcStruct) rows.next(row); + assertEquals(Timestamp.valueOf("1900-01-01 00:00:00"), + row.getFieldValue(0)); + assertEquals(new IntWritable(200000), union.getObject()); + for(int i=1900; i < 2200; ++i) { + row = (OrcStruct) rows.next(row); + assertEquals(Timestamp.valueOf(i + "-05-05 12:34:56." + i), + row.getFieldValue(0)); + if ((i & 1) == 0) { + assertEquals(0, union.getTag()); + assertEquals(new IntWritable(i*i), union.getObject()); + } else { + assertEquals(1, union.getTag()); + assertEquals(new Text(new Integer(i*i).toString()), union.getObject()); + } + } + for(int i=0; i < 1000; ++i) { + row = (OrcStruct) rows.next(row); + assertEquals(new IntWritable(1732050807), union.getObject()); + } + row = (OrcStruct) rows.next(row); + assertEquals(new IntWritable(0), union.getObject()); + row = (OrcStruct) rows.next(row); + assertEquals(new IntWritable(10), union.getObject()); + row = (OrcStruct) rows.next(row); + assertEquals(new IntWritable(138), union.getObject()); + assertEquals(false, rows.hasNext()); + assertEquals(1.0, rows.getProgress(), 0.00001); + assertEquals(reader.getNumberOfRows(), rows.getRowNumber()); + rows.close(); + } + + /** + * Read and write a randomly generated snappy file. + * @throws Exception + */ + @Test + public void testSnappy() throws Exception { + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + ObjectInspector inspector = + ObjectInspectorFactory.getReflectionObjectInspector(InnerStruct.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Writer writer = OrcFile.createWriter(fs, p, inspector, + 1000, CompressionKind.SNAPPY, 100, 10000); + Random rand = new Random(12); + for(int i=0; i < 10000; ++i) { + writer.addRow(new InnerStruct(rand.nextInt(), + Integer.toHexString(rand.nextInt()))); + } + writer.close(); + Reader reader = OrcFile.createReader(fs, p); + RecordReader rows = reader.rows(null); + rand = new Random(12); + OrcStruct row = null; + for(int i=0; i < 10000; ++i) { + assertEquals(true, rows.hasNext()); + row = (OrcStruct) rows.next(row); + assertEquals(rand.nextInt(), ((IntWritable) row.getFieldValue(0)).get()); + assertEquals(Integer.toHexString(rand.nextInt()), + row.getFieldValue(1).toString()); + } + assertEquals(false, rows.hasNext()); + rows.close(); + } + + /** + * Read and write a randomly generated snappy file. + * @throws Exception + */ + @Test + public void testWithoutIndex() throws Exception { + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + ObjectInspector inspector = + ObjectInspectorFactory.getReflectionObjectInspector(InnerStruct.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Writer writer = OrcFile.createWriter(fs, p, inspector, + 5000, CompressionKind.SNAPPY, 1000, 0); + Random rand = new Random(24); + for(int i=0; i < 10000; ++i) { + InnerStruct row = new InnerStruct(rand.nextInt(), + Integer.toBinaryString(rand.nextInt())); + for(int j=0; j< 5; ++j) { + writer.addRow(row); + } + } + writer.close(); + Reader reader = OrcFile.createReader(fs, p); + assertEquals(50000, reader.getNumberOfRows()); + assertEquals(0, reader.getRowIndexStride()); + StripeInformation stripe = reader.getStripes().iterator().next(); + assertEquals(true, stripe.getDataLength() != 0); + assertEquals(0, stripe.getIndexLength()); + RecordReader rows = reader.rows(null); + rand = new Random(24); + OrcStruct row = null; + for(int i=0; i < 10000; ++i) { + int intVal = rand.nextInt(); + String strVal = Integer.toBinaryString(rand.nextInt()); + for(int j=0; j < 5; ++j) { + assertEquals(true, rows.hasNext()); + row = (OrcStruct) rows.next(row); + assertEquals(intVal, ((IntWritable) row.getFieldValue(0)).get()); + assertEquals(strVal, row.getFieldValue(1).toString()); + } + } + assertEquals(false, rows.hasNext()); + rows.close(); + } +} Index: contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestDynamicArray.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestDynamicArray.java (revision 0) +++ contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestDynamicArray.java (working copy) @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestDynamicArray { + + @Test + public void testByteArray() throws Exception { + DynamicByteArray dba = new DynamicByteArray(3, 10); + dba.add((byte) 0); + dba.add((byte) 1); + dba.set(3, (byte) 3); + dba.set(2, (byte) 2); + dba.add((byte) 4); + assertEquals("{0,1,2,3,4}", dba.toString()); + assertEquals(5, dba.size()); + byte[] val = new byte[0]; + assertEquals(0, dba.compare(val, 0, 0, 2, 0)); + assertEquals(-1, dba.compare(val, 0, 0, 2, 1)); + val = new byte[]{3,42}; + assertEquals(1, dba.compare(val, 0, 1, 2, 0)); + assertEquals(1, dba.compare(val, 0, 1, 2, 1)); + assertEquals(0, dba.compare(val, 0, 1, 3, 1)); + assertEquals(-1, dba.compare(val, 0, 1, 3, 2)); + assertEquals(1, dba.compare(val, 0, 2, 3, 1)); + val = new byte[256]; + for(int b=-128; b < 128; ++b) { + dba.add((byte) b); + val[b+128] = (byte) b; + } + assertEquals(0, dba.compare(val, 0, 256, 5, 256)); + assertEquals(1, dba.compare(val, 0, 1, 0, 1)); + assertEquals(1, dba.compare(val, 254, 1, 0, 1)); + assertEquals(1, dba.compare(val, 120, 1, 64, 1)); + } + + @Test + public void testIntArray() throws Exception { + DynamicIntArray dia = new DynamicIntArray(10); + for(int i=0; i < 10000; ++i) { + dia.add(2*i); + } + assertEquals(10000, dia.size()); + for(int i=0; i < 10000; ++i) { + assertEquals(2*i, dia.get(i)); + } + dia.clear(); + assertEquals(0, dia.size()); + dia.add(3); + dia.add(12); + dia.add(65); + assertEquals("{3,12,65}", dia.toString()); + for(int i=0; i < 5; ++i) { + dia.increment(i, 3); + } + assertEquals("{6,15,68,3,3}", dia.toString()); + } +} Index: contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java (revision 0) +++ contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java (working copy) @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.PrintStream; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TestFileDump { + + Path workDir = new Path(System.getProperty("test.tmp.dir", + "target/test/tmp")); + Path resourceDir = new Path(System.getProperty("test.resources.dir", + "src/test/resources")); + + static class MyRecord { + float f; + double d; + MyRecord(float f, double d) { + this.f = f; + this.d = d; + } + } + + private static final String outputFilename = "/orc-file-dump.out"; + + private static void checkOutput(String expected, + String actual) throws Exception { + BufferedReader eStream = + new BufferedReader(new FileReader(expected)); + BufferedReader aStream = + new BufferedReader(new FileReader(actual)); + String line = eStream.readLine(); + while (line != null) { + assertEquals(line, aStream.readLine()); + line = eStream.readLine(); + } + assertNull(eStream.readLine()); + assertNull(aStream.readLine()); + } + + @Test + public void testDump() throws Exception { + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + ObjectInspector inspector = + ObjectInspectorFactory.getReflectionObjectInspector(MyRecord.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Writer writer = OrcFile.createWriter(fs, p, inspector, + 100000, CompressionKind.ZLIB, 10000, 10000); + Random r1 = new Random(1); + for(int i=0; i < 21000; ++i) { + writer.addRow(new MyRecord(r1.nextFloat(), r1.nextDouble())); + } + writer.close(); + PrintStream origOut = System.out; + FileOutputStream myOut = new FileOutputStream(workDir + + "/orc-file-dump.out"); + + // replace stdout and run command + System.setOut(new PrintStream(myOut)); + FileDump.main(new String[]{p.toString()}); + System.out.flush(); + System.setOut(origOut); + + checkOutput(resourceDir + outputFilename, workDir + outputFilename); + } +} Index: contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcStruct.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcStruct.java (revision 0) +++ contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcStruct.java (working copy) @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class TestOrcStruct { + + @Test + public void testStruct() throws Exception { + OrcStruct st1 = new OrcStruct(4); + OrcStruct st2 = new OrcStruct(4); + OrcStruct st3 = new OrcStruct(3); + st1.setFieldValue(0, "hop"); + st1.setFieldValue(1, "on"); + st1.setFieldValue(2, "pop"); + st1.setFieldValue(3, 42); + assertEquals(false, st1.equals(null)); + st2.setFieldValue(0, "hop"); + st2.setFieldValue(1, "on"); + st2.setFieldValue(2, "pop"); + st2.setFieldValue(3, 42); + assertEquals(st1, st2); + st3.setFieldValue(0, "hop"); + st3.setFieldValue(1, "on"); + st3.setFieldValue(2, "pop"); + assertEquals(false, st1.equals(st3)); + assertEquals(11241, st1.hashCode()); + assertEquals(st1.hashCode(), st2.hashCode()); + assertEquals(11204, st3.hashCode()); + assertEquals("{hop, on, pop, 42}", st1.toString()); + st1.setFieldValue(3, null); + assertEquals(false, st1.equals(st2)); + assertEquals(false, st2.equals(st1)); + st2.setFieldValue(3, null); + assertEquals(st1, st2); + } + + @Test + public void testInspectorFromTypeInfo() throws Exception { + TypeInfo typeInfo = + TypeInfoUtils.getTypeInfoFromTypeString("struct,c11:map,c12:uniontype" + + ",c13:array>"); + StructObjectInspector inspector = (StructObjectInspector) + OrcStruct.createObjectInspector(typeInfo); + assertEquals("struct,c11:map,c12:union{int},c13:list>", + inspector.getTypeName()); + assertEquals(null, + inspector.getAllStructFieldRefs().get(0).getFieldComment()); + assertEquals(null, inspector.getStructFieldRef("UNKNOWN")); + OrcStruct s1 = new OrcStruct(13); + for(int i=0; i < 13; ++i) { + s1.setFieldValue(i, i); + } + + List list = new ArrayList(); + list.addAll(Arrays.asList(0,1,2,3,4,5,6,7,8,9,10,11,12)); + assertEquals(list, inspector.getStructFieldsDataAsList(s1)); + ListObjectInspector listOI = (ListObjectInspector) + inspector.getAllStructFieldRefs().get(12).getFieldObjectInspector(); + assertEquals(ObjectInspector.Category.LIST, listOI.getCategory()); + assertEquals(10, listOI.getListElement(list, 10)); + assertEquals(13, listOI.getListLength(list)); + + Map map = new HashMap(); + map.put(1,2); + map.put(2,4); + map.put(3,6); + MapObjectInspector mapOI = (MapObjectInspector) + inspector.getAllStructFieldRefs().get(10).getFieldObjectInspector(); + assertEquals(3, mapOI.getMapSize(map)); + assertEquals(4, mapOI.getMapValueElement(map, 2)); + } + + @Test + public void testUnion() throws Exception { + OrcUnion un1 = new OrcUnion(); + OrcUnion un2 = new OrcUnion(); + un1.set((byte) 0, "hi"); + un2.set((byte) 0, "hi"); + assertEquals(un1, un2); + assertEquals(un1.hashCode(), un2.hashCode()); + un2.set((byte) 0, null); + assertEquals(false, un1.equals(un2)); + assertEquals(false, un2.equals(un1)); + un1.set((byte) 0, null); + assertEquals(un1, un2); + un2.set((byte) 0, "hi"); + un1.set((byte) 1, "hi"); + assertEquals(false, un1.equals(un2)); + assertEquals(false, un1.hashCode() == un2.hashCode()); + un2.set((byte) 1, "byte"); + assertEquals(false, un1.equals(un2)); + assertEquals("union(1, hi)", un1.toString()); + assertEquals(false, un1.equals(null)); + } +} Index: contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (revision 0) +++ contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (working copy) @@ -0,0 +1,348 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.hive.ql.io.InputFormatChecker; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.mapred.Reporter; +import org.junit.Test; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +public class TestInputOutputFormat { + + Path workDir = new Path(System.getProperty("test.tmp.dir","target/test/tmp")); + + public static class MyRow implements Writable { + int x; + int y; + MyRow(int x, int y) { + this.x = x; + this.y = y; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + throw new UnsupportedOperationException("no write"); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + throw new UnsupportedOperationException("no read"); + } + } + + @Test + public void test1() throws Exception { + JobConf conf = new JobConf(); + Properties properties = new Properties(); + FileSystem fs = FileSystem.getLocal(conf); + StructObjectInspector inspector = (StructObjectInspector) + ObjectInspectorFactory.getReflectionObjectInspector(MyRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + SerDe serde = new OrcSerde(); + HiveOutputFormat outFormat = new OrcOutputFormat(); + FileSinkOperator.RecordWriter writer = + outFormat.getHiveRecordWriter(conf, p, MyRow.class, true, + properties, Reporter.NULL); + writer.write(serde.serialize(new MyRow(1,2), inspector)); + writer.write(serde.serialize(new MyRow(2,2), inspector)); + writer.write(serde.serialize(new MyRow(3,2), inspector)); + writer.close(true); + serde = new OrcSerde(); + properties.setProperty("columns", "x,y"); + properties.setProperty("columns.types", "int:int"); + serde.initialize(conf, properties); + assertEquals(OrcSerde.OrcSerdeRow.class, serde.getSerializedClass()); + inspector = (StructObjectInspector) serde.getObjectInspector(); + assertEquals("struct", inspector.getTypeName()); + InputFormat in = new OrcInputFormat(); + FileInputFormat.setInputPaths(conf, p.toString()); + InputSplit[] splits = in.getSplits(conf, 1); + assertEquals(1, splits.length); + + // the the validate input method + ArrayList fileList = new ArrayList(); + assertEquals(false, + ((InputFormatChecker) in).validateInput(fs, new HiveConf(), fileList)); + fileList.add(fs.getFileStatus(p)); + assertEquals(true, + ((InputFormatChecker) in).validateInput(fs, new HiveConf(), fileList)); + fileList.add(fs.getFileStatus(workDir)); + assertEquals(false, + ((InputFormatChecker) in).validateInput(fs, new HiveConf(), fileList)); + + + // read the whole file + org.apache.hadoop.mapred.RecordReader reader = + in.getRecordReader(splits[0], conf, Reporter.NULL); + Object key = reader.createKey(); + Writable value = (Writable) reader.createValue(); + int rowNum = 0; + List fields =inspector.getAllStructFieldRefs(); + IntObjectInspector intInspector = + (IntObjectInspector) fields.get(0).getFieldObjectInspector(); + assertEquals(0.0, reader.getProgress(), 0.00001); + assertEquals(0, reader.getPos()); + while (reader.next(key, value)) { + assertEquals(++rowNum, intInspector.get(inspector. + getStructFieldData(serde.deserialize(value), fields.get(0)))); + assertEquals(2, intInspector.get(inspector. + getStructFieldData(serde.deserialize(value), fields.get(1)))); + } + assertEquals(3, rowNum); + assertEquals(1.0, reader.getProgress(), 0.00001); + reader.close(); + + // read just the first column + conf.set("hive.io.file.readcolumn.ids", "0"); + reader = in.getRecordReader(splits[0], conf, Reporter.NULL); + key = reader.createKey(); + value = (Writable) reader.createValue(); + rowNum = 0; + fields = inspector.getAllStructFieldRefs(); + while (reader.next(key, value)) { + assertEquals(++rowNum, intInspector.get(inspector. + getStructFieldData(value, fields.get(0)))); + assertEquals(null, inspector.getStructFieldData(value, fields.get(1))); + } + assertEquals(3, rowNum); + reader.close(); + } + + static class NestedRow implements Writable { + int z; + MyRow r; + NestedRow(int x, int y, int z) { + this.z = z; + this.r = new MyRow(x,y); + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + throw new UnsupportedOperationException("unsupported"); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + throw new UnsupportedOperationException("unsupported"); + } + } + + @Test + public void testMROutput() throws Exception { + JobConf conf = new JobConf(); + Properties properties = new Properties(); + FileSystem fs = FileSystem.getLocal(conf); + StructObjectInspector inspector = (StructObjectInspector) + ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + SerDe serde = new OrcSerde(); + OutputFormat outFormat = new OrcOutputFormat(); + RecordWriter writer = + outFormat.getRecordWriter(fs, conf, p.toString(), Reporter.NULL); + writer.write(NullWritable.get(), + serde.serialize(new NestedRow(1,2,3), inspector)); + writer.write(NullWritable.get(), + serde.serialize(new NestedRow(4,5,6), inspector)); + writer.write(NullWritable.get(), + serde.serialize(new NestedRow(7,8,9), inspector)); + writer.close(Reporter.NULL); + serde = new OrcSerde(); + properties.setProperty("columns", "z,r"); + properties.setProperty("columns.types", "int:struct"); + serde.initialize(conf, properties); + inspector = (StructObjectInspector) serde.getObjectInspector(); + InputFormat in = new OrcInputFormat(); + FileInputFormat.setInputPaths(conf, p.toString()); + InputSplit[] splits = in.getSplits(conf, 1); + assertEquals(1, splits.length); + conf.set("hive.io.file.readcolumn.ids", "1"); + org.apache.hadoop.mapred.RecordReader reader = + in.getRecordReader(splits[0], conf, Reporter.NULL); + Object key = reader.createKey(); + Object value = reader.createValue(); + int rowNum = 0; + List fields = inspector.getAllStructFieldRefs(); + StructObjectInspector inner = (StructObjectInspector) + fields.get(1).getFieldObjectInspector(); + List inFields = inner.getAllStructFieldRefs(); + IntObjectInspector intInspector = + (IntObjectInspector) fields.get(0).getFieldObjectInspector(); + while (reader.next(key, value)) { + assertEquals(null, inspector.getStructFieldData(value, fields.get(0))); + Object sub = inspector.getStructFieldData(value, fields.get(1)); + assertEquals(3*rowNum+1, intInspector.get(inner.getStructFieldData(sub, + inFields.get(0)))); + assertEquals(3*rowNum+2, intInspector.get(inner.getStructFieldData(sub, + inFields.get(1)))); + rowNum += 1; + } + assertEquals(3, rowNum); + reader.close(); + + } + + @Test + public void testEmptyFile() throws Exception { + JobConf conf = new JobConf(); + Properties properties = new Properties(); + FileSystem fs = FileSystem.getLocal(conf); + ObjectInspector inspector = + ObjectInspectorFactory.getReflectionObjectInspector(MyRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + HiveOutputFormat outFormat = new OrcOutputFormat(); + FileSinkOperator.RecordWriter writer = + outFormat.getHiveRecordWriter(conf, p, MyRow.class, true, + properties, Reporter.NULL); + writer.close(true); + properties.setProperty("columns", "x,y"); + properties.setProperty("columns.types", "int:int"); + SerDe serde = new OrcSerde(); + serde.initialize(conf, properties); + InputFormat in = new OrcInputFormat(); + FileInputFormat.setInputPaths(conf, p.toString()); + InputSplit[] splits = in.getSplits(conf, 1); + assertEquals(1, splits.length); + + // read the whole file + conf.set("hive.io.file.readcolumn.ids", "0,1"); + org.apache.hadoop.mapred.RecordReader reader = + in.getRecordReader(splits[0], conf, Reporter.NULL); + Object key = reader.createKey(); + Object value = reader.createValue(); + assertEquals(0.0, reader.getProgress(), 0.00001); + assertEquals(0, reader.getPos()); + assertEquals(false, reader.next(key, value)); + reader.close(); + assertEquals(null, serde.getSerDeStats()); + } + + static class StringRow implements Writable { + String str; + String str2; + StringRow(String s) { + str = s; + str2 = s; + } + @Override + public void write(DataOutput dataOutput) throws IOException { + throw new UnsupportedOperationException("no write"); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + throw new UnsupportedOperationException("no read"); + } + } + + @Test + public void testDefaultTypes() throws Exception { + JobConf conf = new JobConf(); + Properties properties = new Properties(); + FileSystem fs = FileSystem.getLocal(conf); + StructObjectInspector inspector = (StructObjectInspector) + ObjectInspectorFactory.getReflectionObjectInspector(StringRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + Path p = new Path(workDir, "file.orc"); + fs.delete(p, false); + SerDe serde = new OrcSerde(); + HiveOutputFormat outFormat = new OrcOutputFormat(); + FileSinkOperator.RecordWriter writer = + outFormat.getHiveRecordWriter(conf, p, StringRow.class, true, + properties, Reporter.NULL); + writer.write(serde.serialize(new StringRow("owen"), inspector)); + writer.write(serde.serialize(new StringRow("beth"), inspector)); + writer.write(serde.serialize(new StringRow("laurel"), inspector)); + writer.write(serde.serialize(new StringRow("hazen"), inspector)); + writer.write(serde.serialize(new StringRow("colin"), inspector)); + writer.write(serde.serialize(new StringRow("miles"), inspector)); + writer.close(true); + serde = new OrcSerde(); + properties.setProperty("columns", "str,str2"); + serde.initialize(conf, properties); + inspector = (StructObjectInspector) serde.getObjectInspector(); + assertEquals("struct", inspector.getTypeName()); + InputFormat in = new OrcInputFormat(); + FileInputFormat.setInputPaths(conf, p.toString()); + InputSplit[] splits = in.getSplits(conf, 1); + assertEquals(1, splits.length); + + // read the whole file + org.apache.hadoop.mapred.RecordReader reader = + in.getRecordReader(splits[0], conf, Reporter.NULL); + Object key = reader.createKey(); + Writable value = (Writable) reader.createValue(); + List fields =inspector.getAllStructFieldRefs(); + StringObjectInspector strInspector = (StringObjectInspector) + fields.get(0).getFieldObjectInspector(); + assertEquals(true, reader.next(key, value)); + assertEquals("owen", strInspector.getPrimitiveJavaObject(inspector. + getStructFieldData(value, fields.get(0)))); + assertEquals(true, reader.next(key, value)); + assertEquals("beth", strInspector.getPrimitiveJavaObject(inspector. + getStructFieldData(value, fields.get(0)))); + assertEquals(true, reader.next(key, value)); + assertEquals("laurel", strInspector.getPrimitiveJavaObject(inspector. + getStructFieldData(value, fields.get(0)))); + assertEquals(true, reader.next(key, value)); + assertEquals("hazen", strInspector.getPrimitiveJavaObject(inspector. + getStructFieldData(value, fields.get(0)))); + assertEquals(true, reader.next(key, value)); + assertEquals("colin", strInspector.getPrimitiveJavaObject(inspector. + getStructFieldData(value, fields.get(0)))); + assertEquals(true, reader.next(key, value)); + assertEquals("miles", strInspector.getPrimitiveJavaObject(inspector. + getStructFieldData(value, fields.get(0)))); + assertEquals(false, reader.next(key, value)); + reader.close(); + } +} Index: contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java (revision 0) +++ contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java (working copy) @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestStreamName { + + @Test + public void test1() throws Exception { + StreamName s1 = new StreamName(3, OrcProto.Stream.Kind.DATA); + StreamName s2 = new StreamName(3, + OrcProto.Stream.Kind.DICTIONARY_DATA); + StreamName s3 = new StreamName(5, OrcProto.Stream.Kind.DATA); + StreamName s4 = new StreamName(5, + OrcProto.Stream.Kind.DICTIONARY_DATA); + StreamName s1p = new StreamName(3, OrcProto.Stream.Kind.DATA); + assertEquals(true, s1.equals(s1)); + assertEquals(false, s1.equals(s2)); + assertEquals(false, s1.equals(s3)); + assertEquals(true, s1.equals(s1p)); + assertEquals(true, s1.compareTo(null) < 0); + assertEquals(false, s1.equals(null)); + assertEquals(true, s1.compareTo(s2) < 0); + assertEquals(true, s2.compareTo(s3) < 0); + assertEquals(true, s3.compareTo(s4) < 0); + assertEquals(true, s4.compareTo(s1p) > 0); + assertEquals(0, s1p.compareTo(s1)); + } +} Index: contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java =================================================================== --- contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java (revision 0) +++ contrib/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java (working copy) @@ -0,0 +1,273 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import java.io.BufferedOutputStream; +import java.io.IOException; + +import static junit.framework.Assert.assertEquals; + +/** + * Test the red-black tree with string keys. + */ +public class TestStringRedBlackTree { + + /** + * Checks the red-black tree rules to make sure that we have correctly built + * a valid tree. + * + * Properties: + * 1. Red nodes must have black children + * 2. Each node must have the same black height on both sides. + * + * @param node The id of the root of the subtree to check for the red-black + * tree properties. + * @return The black-height of the subtree. + */ + private int checkSubtree(RedBlackTree tree, int node, IntWritable count + ) throws IOException { + if (node == RedBlackTree.NULL) { + return 1; + } + count.set(count.get() + 1); + boolean is_red = tree.isRed(node); + int left = tree.getLeft(node); + int right = tree.getRight(node); + if (is_red) { + if (tree.isRed(left)) { + printTree(tree, "", tree.root); + throw new IllegalStateException("Left node of " + node + " is " + left + + " and both are red."); + } + if (tree.isRed(right)) { + printTree(tree, "", tree.root); + throw new IllegalStateException("Right node of " + node + " is " + + right + " and both are red."); + } + } + int left_depth = checkSubtree(tree, left, count); + int right_depth = checkSubtree(tree, right, count); + if (left_depth != right_depth) { + printTree(tree, "", tree.root); + throw new IllegalStateException("Lopsided tree at node " + node + + " with depths " + left_depth + " and " + right_depth); + } + if (is_red) { + return left_depth; + } else { + return left_depth + 1; + } + } + + /** + * Checks the validity of the entire tree. Also ensures that the number of + * nodes visited is the same as the size of the set. + */ + void checkTree(RedBlackTree tree) throws IOException { + IntWritable count = new IntWritable(0); + if (tree.isRed(tree.root)) { + printTree(tree, "", tree.root); + throw new IllegalStateException("root is red"); + } + checkSubtree(tree, tree.root, count); + if (count.get() != tree.size) { + printTree(tree, "", tree.root); + throw new IllegalStateException("Broken tree! visited= " + count.get() + + " size=" + tree.size); + } + } + + void printTree(RedBlackTree tree, String indent, int node + ) throws IOException { + if (node == RedBlackTree.NULL) { + System.err.println(indent + "NULL"); + } else { + System.err.println(indent + "Node " + node + " color " + + (tree.isRed(node) ? "red" : "black") + " count " + tree.getCount(node)); + printTree(tree, indent + " ", tree.getLeft(node)); + printTree(tree, indent + " ", tree.getRight(node)); + } + } + + private static class MyVisitor implements StringRedBlackTree.Visitor { + private final String[] words; + private final int[] counts; + private final int[] order; + private final DataOutputBuffer buffer = new DataOutputBuffer(); + int current = 0; + + MyVisitor(String[] args, int[] counts, int[] order) { + words = args; + this.counts = counts; + this.order = order; + } + + @Override + public void visit(StringRedBlackTree.VisitorContext context + ) throws IOException { + String word = context.getText().toString(); + assertEquals("in word " + current, words[current], word); + assertEquals("in word " + current, counts[current], context.getCount()); + assertEquals("in word " + current, order[current], + context.getOriginalPosition()); + buffer.reset(); + context.writeBytes(buffer); + assertEquals(word, new String(buffer.getData(),0,buffer.getLength())); + current += 1; + } + } + + void checkContents(StringRedBlackTree tree, int[] counts, int[] order, + String... params + ) throws IOException { + tree.visit(new MyVisitor(params, counts, order)); + } + + StringRedBlackTree buildTree(String... params) throws IOException { + StringRedBlackTree result = new StringRedBlackTree(); + for(String word: params) { + result.add(word); + checkTree(result); + } + return result; + } + + @Test + public void test1() throws Exception { + StringRedBlackTree tree = new StringRedBlackTree(5); + assertEquals(0, tree.getByteSize()); + checkTree(tree); + assertEquals(0, tree.add("owen")); + checkTree(tree); + assertEquals(1, tree.add("ashutosh")); + checkTree(tree); + assertEquals(0, tree.add("owen")); + checkTree(tree); + assertEquals(2, tree.add("alan")); + checkTree(tree); + assertEquals(2, tree.add("alan")); + checkTree(tree); + assertEquals(1, tree.add("ashutosh")); + checkTree(tree); + assertEquals(3, tree.add("greg")); + checkTree(tree); + assertEquals(4, tree.add("eric")); + checkTree(tree); + assertEquals(5, tree.add("arun")); + checkTree(tree); + assertEquals(6, tree.size()); + checkTree(tree); + assertEquals(6, tree.add("eric14")); + checkTree(tree); + assertEquals(7, tree.add("o")); + checkTree(tree); + assertEquals(8, tree.add("ziggy")); + checkTree(tree); + assertEquals(9, tree.add("z")); + checkTree(tree); + checkContents(tree, new int[]{2,1,2,1,1,1,1,2,1,1}, + new int[]{2,5,1,4,6,3,7,0,9,8}, + "alan", "arun", "ashutosh", "eric", "eric14", "greg", + "o", "owen", "z", "ziggy"); + assertEquals(10*5*4 + 8 + 6 + 5 + 5 * 4 + 2 * 1, tree.getByteSize()); + // check that adding greg again bumps the count + assertEquals(1, tree.getCount(3)); + assertEquals(3, tree.add("greg")); + assertEquals(2, tree.getCount(3)); + assertEquals(41, tree.getCharacterSize()); + // add some more strings to test the different branches of the + // rebalancing + assertEquals(10, tree.add("zak")); + checkTree(tree); + assertEquals(11, tree.add("eric1")); + checkTree(tree); + assertEquals(12, tree.add("ash")); + checkTree(tree); + assertEquals(13, tree.add("harry")); + checkTree(tree); + assertEquals(14, tree.add("john")); + checkTree(tree); + tree.clear(); + checkTree(tree); + assertEquals(0, tree.getByteSize()); + assertEquals(0, tree.getCharacterSize()); + } + + @Test + public void test2() throws Exception { + StringRedBlackTree tree = + buildTree("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", + "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"); + assertEquals(26, tree.size()); + checkContents(tree, new int[]{1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1, + 1,1,1, 1,1,1, 1,1}, new int[]{0,1,2, 3,4,5, 6,7,8, 9,10,11, 12,13,14, + 15,16,17, 18,19,20, 21,22,23, 24,25}, + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j","k", "l", "m", "n", "o", + "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"); + } + + @Test + public void test3() throws Exception { + StringRedBlackTree tree = + buildTree("z", "y", "x", "w", "v", "u", "t", "s", "r", "q", "p", "o", "n", + "m", "l", "k", "j", "i", "h", "g", "f", "e", "d", "c", "b", "a"); + assertEquals(26, tree.size()); + checkContents(tree, new int[]{1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1, + 1,1,1, 1,1,1, 1,1}, new int[]{25,24,23, 22,21,20, 19,18,17, 16,15,14, + 13,12,11, 10,9,8, 7,6,5, 4,3,2, 1,0}, + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", + "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"); + } + + public static void main(String[] args) throws Exception { + TestStringRedBlackTree test = new TestStringRedBlackTree(); + test.test1(); + test.test2(); + test.test3(); + TestOrcFile test1 = new TestOrcFile(); + test1.test1(); + test1.emptyFile(); + test1.metaData(); + test1.testUnionAndTimestamp(); + test1.columnProjection(); + test1.testSnappy(); + test1.testWithoutIndex(); + TestFileDump test2 = new TestFileDump(); + test2.testDump(); + TestStreamName test3 = new TestStreamName(); + test3.test1(); + TestInputOutputFormat test4 = new TestInputOutputFormat(); + test4.test1(); + test4.testMROutput(); + test4.testEmptyFile(); + test4.testDefaultTypes(); + TestOrcStruct test5 = new TestOrcStruct(); + test5.testStruct(); + test5.testInspectorFromTypeInfo(); + test5.testUnion(); + TestDynamicArray test6 = new TestDynamicArray(); + test6.testByteArray(); + test6.testIntArray(); + } +} Index: contrib/src/test/resources/orc-file-dump.out =================================================================== --- contrib/src/test/resources/orc-file-dump.out (revision 0) +++ contrib/src/test/resources/orc-file-dump.out (working copy) @@ -0,0 +1,36 @@ +Structure for target/test/tmp/file.orc +Rows: 21000 +Compression: ZLIB +Compression size: 10000 +Type: struct + +Statistics: + Column 0: count: 21000 + Column 1: count: 21000 min: 8.881092071533203E-6 max: 0.9999387264251709 sum: 10551.349509894848 + Column 2: count: 21000 min: 2.3379546596924428E-5 max: 0.9999973837551178 sum: 10486.848248849014 + +Stripes: + Stripe: offset: 3 data: 111528 rows: 10001 tail: 52 index: 166 + Stream: column 0 section ROW_INDEX start: 3 length 16 + Stream: column 1 section ROW_INDEX start: 19 length 70 + Stream: column 2 section ROW_INDEX start: 89 length 80 + Stream: column 1 section PRESENT start: 169 length 14 + Stream: column 1 section DATA start: 183 length 35917 + Stream: column 2 section PRESENT start: 36100 length 14 + Stream: column 2 section DATA start: 36114 length 75583 + Stripe: offset: 111749 data: 111478 rows: 10001 tail: 52 index: 180 + Stream: column 0 section ROW_INDEX start: 111749 length 16 + Stream: column 1 section ROW_INDEX start: 111765 length 79 + Stream: column 2 section ROW_INDEX start: 111844 length 85 + Stream: column 1 section PRESENT start: 111929 length 14 + Stream: column 1 section DATA start: 111943 length 35862 + Stream: column 2 section PRESENT start: 147805 length 14 + Stream: column 2 section DATA start: 147819 length 75588 + Stripe: offset: 223459 data: 11174 rows: 998 tail: 50 index: 107 + Stream: column 0 section ROW_INDEX start: 223459 length 10 + Stream: column 1 section ROW_INDEX start: 223469 length 48 + Stream: column 2 section ROW_INDEX start: 223517 length 49 + Stream: column 1 section PRESENT start: 223566 length 7 + Stream: column 1 section DATA start: 223573 length 3607 + Stream: column 2 section PRESENT start: 227180 length 7 + Stream: column 2 section DATA start: 227187 length 7553 Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java (working copy) @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +public interface StringColumnStatistics extends ColumnStatistics { + String getMinimum(); + + String getMaximum(); +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java (working copy) @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.nio.ByteBuffer; + +interface CompressionCodec { + /** + * Compress the in buffer to the out buffer. + * @param in the bytes to compress + * @param out the uncompressed bytes + * @param overflow put any additional bytes here + * @return true if the output is smaller than input + * @throws IOException + */ + boolean compress(ByteBuffer in, ByteBuffer out, ByteBuffer overflow + ) throws IOException; + + /** + * Decompress the in buffer to the out buffer. + * @param in the bytes to decompress + * @param out the decompressed bytes + * @throws IOException + */ + void decompress(ByteBuffer in, ByteBuffer out) throws IOException; +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java (working copy) @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +public interface Reader { + + /** + * Get the number of rows in the file + * @return the number of rows + */ + public long getNumberOfRows(); + + /** + * Get the user metadata keys + * @return the set of metadata keys + */ + public Iterable getMetadataKeys(); + + /** + * Get a user metadata value + * @param key a key given by the user + * @return the bytes associated with the given key + */ + public ByteBuffer getMetadataValue(String key); + + /** + * Get the compression kind. + * @return the kind of compression in the file + */ + public CompressionKind getCompression(); + + /** + * Get the buffer size for the compression. + * @return number of bytes to buffer for the compression codec. + */ + public int getCompressionSize(); + + /** + * Get the number of rows per a entry in the row index + * @return the number of rows per an entry in the row index or 0 if there + * is no row index. + */ + public int getRowIndexStride(); + + /** + * Get the list of stripes. + * @return the information about the stripes in order + */ + public Iterable getStripes(); + + /** + * Get the object inspector for looking at the objects. + * @return an object inspector for each row returned + */ + public ObjectInspector getObjectInspector(); + + /** + * Get the length of the file. + * @return the number of bytes in the file + */ + public long getContentLength(); + + /** + * Get the statistics about the columns in the file + * @return the information about the column + */ + public ColumnStatistics[] getStatistics(); + + /** + * Get the list of types contained in the file. The root type is the first + * type in the list. + * @return the list of flattened types + */ + public List getTypes(); + + /** + * Create a RecordReader that will scan the entire file. + * @param include true for each column that should be included + * @return A new RecordReader + * @throws IOException + */ + public RecordReader rows(boolean[] include) throws IOException; + + /** + * Create a RecordReader that will start reading at the first stripe after + * offset up to the stripe that starts at offset+length. This is intended + * to work with MapReduce's FileInputFormat where divisions are picked + * blindly, but they must cover all of the rows. + * @param offset a byte offset in the file + * @param length a number of bytes in the file + * @param include true for each column that should be included + * @return a new RecordReader that will read the specified rows. + * @throws IOException + */ + public RecordReader rows(long offset, long length, boolean[] include + ) throws IOException; + +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java (working copy) @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.EOFException; +import java.io.IOException; + +/** + * A reader that reads a sequence of integers. + * */ +class RunLengthIntegerReader { + private final InStream input; + private final boolean signed; + private final long[] literals = + new long[RunLengthIntegerWriter.MAX_LITERAL_SIZE]; + private int numLiterals = 0; + private int delta = 0; + private int used = 0; + private boolean done = false; + private boolean repeat = false; + + RunLengthIntegerReader(InStream input, boolean signed) throws IOException { + this.input = input; + this.signed = signed; + readValues(); + } + + private void readValues() throws IOException { + int control = input.read(); + if (control == -1) { + done = true; + } else if (control < 0x80) { + numLiterals = control + RunLengthIntegerWriter.MIN_REPEAT_SIZE; + used = 0; + repeat = true; + delta = input.read(); + if (delta == -1) { + throw new EOFException("End of stream in RLE Integer"); + } + // convert from 0 to 255 to -128 to 127 by converting to a signed byte + delta = (byte) (0+delta); + if (signed) { + literals[0] = SerializationUtils.readVslong(input); + } else { + literals[0] = SerializationUtils.readVulong(input); + } + } else { + repeat = false; + numLiterals = 0x100 - control; + used = 0; + for(int i=0; i < numLiterals; ++i) { + if (signed) { + literals[i] = SerializationUtils.readVslong(input); + } else { + literals[i] = SerializationUtils.readVulong(input); + } + } + } + } + + boolean hasNext() { + return !done && used != numLiterals; + } + + long next() throws IOException { + long result; + if (repeat) { + result = literals[0] + (used++) * delta; + } else { + result = literals[used++]; + } + if (used == numLiterals) { + readValues(); + } + return result; + } + +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatistics.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatistics.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatistics.java (working copy) @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +public interface ColumnStatistics { + /** + * Get the number of values in this column. It will differ from the number + * of rows because of NULL values and repeated values. + * @return the number of values + */ + long getNumberOfValues(); +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/BooleanColumnStatistics.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/BooleanColumnStatistics.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/BooleanColumnStatistics.java (working copy) @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +public interface BooleanColumnStatistics extends ColumnStatistics { + long getFalseCount(); + + long getTrueCount(); +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java (working copy) @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public interface Writer { + void addUserMetadata(String name, ByteBuffer value); + + void addRow(Object row) throws IOException; + + void close() throws IOException; +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java (working copy) @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +public interface StripeInformation { + /** + * Get the byte offset of the start of the stripe. + * @return the bytes from the start of the file + */ + long getOffset(); + + /** + * Get the length of the stripe's indexes. + * @return the number of bytes in the index + */ + long getIndexLength(); + + /** + * Get the length of the stripe's data + * @return the number of bytes in the stripe + */ + long getDataLength(); + + /** + * Get the length of the stripe's tail section, which contains its index. + * @return the number of bytes in the tail + */ + long getFooterLength(); + + /** + * Get the number of rows in the stripe. + * @return a count of the number of rows + */ + long getNumberOfRows(); +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (working copy) @@ -0,0 +1,195 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.io.InputFormatChecker; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * A MapReduce/Hive input format for ORC files. + */ +public class OrcInputFormat extends FileInputFormat + implements InputFormatChecker { + + private static class OrcRecordReader + implements RecordReader { + private final org.apache.hadoop.hive.ql.io.orc.RecordReader reader; + private final long offset; + private final long length; + private final OrcStruct row; + private boolean firstRow = true; + private float progress = 0.0f; + + OrcRecordReader(Reader file, Configuration conf, + long offset, long length) throws IOException { + this.reader = file.rows(offset, length, + findIncludedColumns(file.getTypes(), conf)); + this.offset = offset; + this.length = length; + if (reader.hasNext()) { + row = (OrcStruct) reader.next(null); + } else { + row = null; + } + } + + @Override + public boolean next(NullWritable key, OrcStruct value) throws IOException { + if (firstRow) { + firstRow = false; + assert value == row: "User didn't pass our value back " + value; + return row != null; + } else if (reader.hasNext()) { + Object obj = reader.next(value); + progress = reader.getProgress(); + assert obj == value : "Reader returned different object " + obj; + return true; + } + return false; + } + + @Override + public NullWritable createKey() { + return NullWritable.get(); + } + + @Override + public OrcStruct createValue() { + return row; + } + + @Override + public long getPos() throws IOException { + return offset + (long) (progress * length); + } + + @Override + public void close() throws IOException { + reader.close(); + } + + @Override + public float getProgress() throws IOException { + return progress; + } + } + + public OrcInputFormat() { + // just set a really small lower bound + setMinSplitSize(16 * 1024); + } + + /** + * Recurse down into a type subtree turning on all of the sub-columns. + * @param types the types of the file + * @param result the global view of columns that should be included + * @param typeId the root of tree to enable + */ + private static void includeColumnRecursive(List types, + boolean[] result, + int typeId) { + result[typeId] = true; + OrcProto.Type type = types.get(typeId); + int children = type.getSubtypesCount(); + for(int i=0; i < children; ++i) { + includeColumnRecursive(types, result, type.getSubtypes(i)); + } + } + + /** + * Take the configuration and figure out which columns we need to include. + * @param types the types of the file + * @param conf the configuration + * @return true for each column that should be included + */ + private static boolean[] findIncludedColumns(List types, + Configuration conf) { + String includedStr = + conf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR); + if (includedStr == null) { + return null; + } else { + int numColumns = types.size(); + boolean[] result = new boolean[numColumns]; + result[0] = true; + OrcProto.Type root = types.get(0); + List included = ColumnProjectionUtils.getReadColumnIDs(conf); + for(int i=1; i < result.length; ++i) { + result[i] = false; + } + for(int i=0; i < root.getSubtypesCount(); ++i) { + if (included.contains(i)) { + includeColumnRecursive(types, result, root.getSubtypes(i)); + } + } + // if we are filtering at least one column, return the boolean array + for(boolean include: result) { + if (!include) { + return result; + } + } + return null; + } + } + + @Override + public RecordReader + getRecordReader(InputSplit inputSplit, JobConf conf, + Reporter reporter) throws IOException { + FileSplit fileSplit = (FileSplit) inputSplit; + Path path = fileSplit.getPath(); + FileSystem fs = path.getFileSystem(conf); + reporter.setStatus(fileSplit.toString()); + return new OrcRecordReader(OrcFile.createReader(fs, path), conf, + fileSplit.getStart(), fileSplit.getLength()); + } + + @Override + public boolean validateInput(FileSystem fs, HiveConf conf, + ArrayList files + ) throws IOException { + if (files.size() <= 0) { + return false; + } + for (FileStatus file : files) { + try { + OrcFile.createReader(fs, file.getPath()); + } catch (IOException e) { + return false; + } + } + return true; + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java (working copy) @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde.OrcSerdeRow; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.FileOutputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordWriter; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.util.Progressable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Properties; + +public class OrcOutputFormat extends FileOutputFormat + implements HiveOutputFormat { + + private static class OrcRecordWriter + implements RecordWriter, + FileSinkOperator.RecordWriter { + private Writer writer = null; + private final FileSystem fs; + private final Path path; + private final Configuration conf; + private final long stripeSize; + private final int compressionSize; + private final CompressionKind compress; + private final int rowIndexStride; + + OrcRecordWriter(FileSystem fs, Path path, Configuration conf, + String stripeSize, String compress, + String compressionSize, String rowIndexStride) { + this.fs = fs; + this.path = path; + this.conf = conf; + this.stripeSize = Long.valueOf(stripeSize); + this.compress = CompressionKind.valueOf(compress); + this.compressionSize = Integer.valueOf(compressionSize); + this.rowIndexStride = Integer.valueOf(rowIndexStride); + } + + @Override + public void write(NullWritable nullWritable, + OrcSerdeRow row) throws IOException { + if (writer == null) { + writer = OrcFile.createWriter(fs, path, row.inspector, stripeSize, + compress, compressionSize, rowIndexStride); + } + writer.addRow(row.realRow); + } + + @Override + public void write(Writable row) throws IOException { + OrcSerdeRow serdeRow = (OrcSerdeRow) row; + if (writer == null) { + writer = OrcFile.createWriter(fs, path, serdeRow.inspector, stripeSize, + compress, compressionSize, rowIndexStride); + } + writer.addRow(serdeRow.realRow); + } + + @Override + public void close(Reporter reporter) throws IOException { + close(true); + } + + @Override + public void close(boolean b) throws IOException { + // if we haven't written any rows, we need to create a file with a + // generic schema. + if (writer == null) { + // a row with no columns + ObjectInspector inspector = ObjectInspectorFactory. + getStandardStructObjectInspector(new ArrayList(), + new ArrayList()); + writer = OrcFile.createWriter(fs, path, inspector, stripeSize, + compress, compressionSize, rowIndexStride); + } + writer.close(); + } + } + + @Override + public RecordWriter + getRecordWriter(FileSystem fileSystem, JobConf conf, String name, + Progressable reporter) throws IOException { + return new OrcRecordWriter(fileSystem, new Path(name), conf, + OrcFile.DEFAULT_STRIPE_SIZE, OrcFile.DEFAULT_COMPRESSION, + OrcFile.DEFAULT_COMPRESSION_BLOCK_SIZE, OrcFile.DEFAULT_ROW_INDEX_STRIDE); + } + + @Override + public FileSinkOperator.RecordWriter + getHiveRecordWriter(JobConf conf, + Path path, + Class valueClass, + boolean isCompressed, + Properties tableProperties, + Progressable reporter) throws IOException { + String stripeSize = tableProperties.getProperty(OrcFile.STRIPE_SIZE, + OrcFile.DEFAULT_STRIPE_SIZE); + String compression = tableProperties.getProperty + (OrcFile.COMPRESSION, OrcFile.DEFAULT_COMPRESSION); + String compressionSize = + tableProperties.getProperty(OrcFile.COMPRESSION_BLOCK_SIZE, + OrcFile.DEFAULT_COMPRESSION_BLOCK_SIZE); + String rowIndexStride = + tableProperties.getProperty(OrcFile.ROW_INDEX_STRIDE, + OrcFile.DEFAULT_ROW_INDEX_STRIDE); + if ("false".equals(tableProperties.getProperty(OrcFile.ENABLE_INDEXES))) { + rowIndexStride = "0"; + } + return new OrcRecordWriter(path.getFileSystem(conf), path, conf, + stripeSize, compression, compressionSize, rowIndexStride); + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java (working copy) @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; + +/** + * A streamFactory that writes a sequence of integers. A control byte is written before + * each run with positive values 0 to 127 meaning 3 to 130 repetitions, each + * repetition is offset by a delta. If the control byte is -1 to -128, 1 to 128 + * literal vint values follow. + */ +class RunLengthIntegerWriter { + static final int MIN_REPEAT_SIZE = 3; + static final int MAX_DELTA = 127; + static final int MIN_DELTA = -128; + static final int MAX_LITERAL_SIZE = 128; + private static final int MAX_REPEAT_SIZE = 127 + MIN_REPEAT_SIZE; + private final PositionedOutputStream output; + private final boolean signed; + private final long[] literals = new long[MAX_LITERAL_SIZE]; + private int numLiterals = 0; + private long delta = 0; + private boolean repeat = false; + private int tailRunLength = 0; + + RunLengthIntegerWriter(PositionedOutputStream output, + boolean signed) { + this.output = output; + this.signed = signed; + } + + private void writeValues() throws IOException { + if (numLiterals != 0) { + if (repeat) { + output.write(numLiterals - MIN_REPEAT_SIZE); + output.write((byte) delta); + if (signed) { + SerializationUtils.writeVslong(output, literals[0]); + } else { + SerializationUtils.writeVulong(output, literals[0]); + } + } else { + output.write(-numLiterals); + for(int i=0; i < numLiterals; ++i) { + if (signed) { + SerializationUtils.writeVslong(output, literals[i]); + } else { + SerializationUtils.writeVulong(output, literals[i]); + } + } + } + repeat = false; + numLiterals = 0; + tailRunLength = 0; + } + } + + void flush() throws IOException { + writeValues(); + output.flush(); + } + + void write(long value) throws IOException { + if (numLiterals == 0) { + literals[numLiterals++] = value; + tailRunLength = 1; + } else if (repeat) { + if (value == literals[0] + delta*numLiterals) { + numLiterals += 1; + if (numLiterals == MAX_REPEAT_SIZE) { + writeValues(); + } + } else { + writeValues(); + literals[numLiterals++] = value; + tailRunLength = 1; + } + } else { + if (tailRunLength == 1) { + delta = value - literals[numLiterals - 1]; + if (delta < MIN_DELTA || delta > MAX_DELTA) { + tailRunLength = 1; + } else { + tailRunLength = 2; + } + } else if (value == literals[numLiterals - 1] + delta) { + tailRunLength += 1; + } else { + delta = value - literals[numLiterals - 1]; + if (delta < MIN_DELTA || delta > MAX_DELTA) { + tailRunLength = 1; + } else { + tailRunLength = 2; + } + } + if (tailRunLength == MIN_REPEAT_SIZE) { + if (numLiterals + 1 == MIN_REPEAT_SIZE) { + repeat = true; + numLiterals += 1; + } else { + numLiterals -= MIN_REPEAT_SIZE - 1; + long base = literals[numLiterals]; + writeValues(); + literals[0] = base; + repeat = true; + numLiterals = MIN_REPEAT_SIZE; + } + } else { + literals[numLiterals++] = value; + if (numLiterals == MAX_LITERAL_SIZE) { + writeValues(); + } + } + } + } + + void getPosition(PositionRecorder recorder) throws IOException { + output.getPosition(recorder); + recorder.addPosition(numLiterals); + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java (working copy) @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.nio.ByteBuffer; + +class OutStream extends PositionedOutputStream { + + interface OutputReceiver { + void output(ByteBuffer buffer) throws IOException; + long getPosition() throws IOException; + } + + static final int HEADER_SIZE = 3; + private final String name; + private final OutputReceiver receiver; + private ByteBuffer compressed = null; + private ByteBuffer overflow = null; + private ByteBuffer current; + private final int bufferSize; + private final CompressionCodec codec; + + OutStream(String name, + int bufferSize, + CompressionCodec codec, + OutputReceiver receiver) throws IOException { + this.name = name; + this.bufferSize = bufferSize; + this.codec = codec; + this.receiver = receiver; + getNewInputBuffer(); + } + + public void clear() throws IOException { + current.position(codec == null ? 0 : HEADER_SIZE); + if (compressed != null) { + compressed.clear(); + } + if (overflow != null) { + overflow.clear(); + } + } + + /** + * Write the length of the compressed bytes. Life is much easier if the + * header is constant length, so just use 3 bytes. Considering most of the + * codecs want between 32k (snappy) and 256k (lzo, zlib), 3 bytes should + * be plenty. We also use the low bit for whether it is the original or + * compressed bytes. + * @param buffer the buffer to write the header to + * @param position the position in the buffer to write at + * @param val the size in the file + * @param original is it uncompressed + */ + private static void writeHeader(ByteBuffer buffer, + int position, + int val, + boolean original) { + buffer.put(position, (byte) ((val << 1) + (original ? 1 : 0))); + buffer.put(position+1, (byte) (val >> 7)); + buffer.put(position+2, (byte) (val >> 15)); + } + + private void getNewInputBuffer() throws IOException { + if (codec == null) { + current = ByteBuffer.allocate(bufferSize); + } else { + current = ByteBuffer.allocate(bufferSize + HEADER_SIZE); + writeHeader(current, 0, bufferSize, true); + current.position(HEADER_SIZE); + } + } + + private ByteBuffer getNewOutputBuffer() throws IOException { + return ByteBuffer.allocate(bufferSize + + (codec == null ? 0 : HEADER_SIZE)); + } + + private void flip() throws IOException { + current.limit(current.position()); + current.position(codec == null ? 0 : HEADER_SIZE); + } + + @Override + public void write(int i) throws IOException { + if (current.remaining() < 1) { + spill(); + } + current.put((byte) i); + } + + @Override + public void write(byte[] bytes, int offset, int length) throws IOException { + int remaining = Math.min(current.remaining(), length); + current.put(bytes, offset, remaining); + length -= remaining; + while (length != 0) { + spill(); + offset += remaining; + remaining = Math.min(current.remaining(), length); + current.put(bytes, offset, remaining); + length -= remaining; + } + } + + private void spill() throws java.io.IOException { + // if there isn't anything in the current buffer, don't spill + if (current.position() == (codec == null ? 0 : HEADER_SIZE)) { + return; + } + flip(); + if (codec == null) { + receiver.output(current); + getNewInputBuffer(); + } else { + if (compressed == null) { + compressed = getNewOutputBuffer(); + } else if (overflow == null) { + overflow = getNewOutputBuffer(); + } + int sizePosn = compressed.position(); + compressed.position(compressed.position()+HEADER_SIZE); + if (codec.compress(current, compressed, overflow)) { + // move position back to after the header + current.position(HEADER_SIZE); + current.limit(current.capacity()); + // find the total bytes in the chunk + int totalBytes = compressed.position() - sizePosn - HEADER_SIZE; + if (overflow != null) { + totalBytes += overflow.position(); + } + writeHeader(compressed, sizePosn, totalBytes, false); + // if we have less than the next header left, spill it. + if (compressed.remaining() < HEADER_SIZE) { + compressed.flip(); + receiver.output(compressed); + compressed = overflow; + overflow = null; + } + } else { + // we are using the original, but need to spill the current + // compressed buffer first. So back up to where we started, + // flip it and add it to done. + if (sizePosn != 0) { + compressed.position(sizePosn); + compressed.flip(); + receiver.output(compressed); + compressed = null; + // if we have an overflow, clear it and make it the new compress + // buffer + if (overflow != null) { + overflow.clear(); + compressed = overflow; + overflow = null; + } + } else { + compressed.clear(); + if (overflow != null) { + overflow.clear(); + } + } + + // now add the current buffer into the done list and get a new one. + current.position(0); + // update the header with the current length + writeHeader(current, 0, current.limit() - HEADER_SIZE, true); + receiver.output(current); + getNewInputBuffer(); + } + } + } + + void getPosition(PositionRecorder recorder) throws IOException { + long doneBytes = receiver.getPosition(); + if (codec == null) { + recorder.addPosition(doneBytes + current.position()); + } else { + recorder.addPosition(doneBytes); + recorder.addPosition(current.position() - HEADER_SIZE); + } + } + + @Override + public void flush() throws IOException { + spill(); + if (compressed != null && compressed.position() != 0) { + compressed.flip(); + receiver.output(compressed); + compressed = null; + } + if (overflow != null && overflow.position() != 0) { + overflow.flip(); + receiver.output(overflow); + overflow = null; + } + } + + @Override + public String toString() { + return name; + } +} + Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java (working copy) @@ -0,0 +1,395 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +final class OrcStruct implements Writable { + + private final Object[] fields; + + OrcStruct(int children) { + fields = new Object[children]; + } + + Object getFieldValue(int fieldIndex) { + return fields[fieldIndex]; + } + + void setFieldValue(int fieldIndex, Object value) { + fields[fieldIndex] = value; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + throw new UnsupportedOperationException("write unsupported"); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + throw new UnsupportedOperationException("readFields unsupported"); + } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != OrcStruct.class) { + return false; + } else { + OrcStruct oth = (OrcStruct) other; + if (fields.length != oth.fields.length) { + return false; + } + for(int i=0; i < fields.length; ++i) { + if (fields[i] == null) { + if (oth.fields[i] != null) { + return false; + } + } else { + if (!fields[i].equals(oth.fields[i])) { + return false; + } + } + } + return true; + } + } + + @Override + public int hashCode() { + int result = fields.length; + for(Object field: fields) { + if (field != null) { + result ^= field.hashCode(); + } + } + return result; + } + + @Override + public String toString() { + StringBuilder buffer = new StringBuilder(); + buffer.append("{"); + for(int i=0; i < fields.length; ++i) { + if (i != 0) { + buffer.append(", "); + } + buffer.append(fields[i]); + } + buffer.append("}"); + return buffer.toString(); + } + + static class Field implements StructField { + private final String name; + private final ObjectInspector inspector; + private final int offset; + + Field(String name, ObjectInspector inspector, int offset) { + this.name = name; + this.inspector = inspector; + this.offset = offset; + } + + @Override + public String getFieldName() { + return name; + } + + @Override + public ObjectInspector getFieldObjectInspector() { + return inspector; + } + + @Override + public String getFieldComment() { + return null; + } + } + + static class OrcStructInspector extends StructObjectInspector { + private final List fields; + + OrcStructInspector(StructTypeInfo info) { + ArrayList fieldNames = info.getAllStructFieldNames(); + ArrayList fieldTypes = info.getAllStructFieldTypeInfos(); + fields = new ArrayList(fieldNames.size()); + for(int i=0; i < fieldNames.size(); ++i) { + fields.add(new Field(fieldNames.get(i), + createObjectInspector(fieldTypes.get(i)), i)); + } + } + + OrcStructInspector(int columnId, List types) { + OrcProto.Type type = types.get(columnId); + int fieldCount = type.getSubtypesCount(); + fields = new ArrayList(fieldCount); + for(int i=0; i < fieldCount; ++i) { + int fieldType = type.getSubtypes(i); + fields.add(new Field(type.getFieldNames(i), + createObjectInspector(fieldType, types), i)); + } + } + + @Override + public List getAllStructFieldRefs() { + return fields; + } + + @Override + public StructField getStructFieldRef(String s) { + for(StructField field: fields) { + if (field.getFieldName().equals(s)) { + return field; + } + } + return null; + } + + @Override + public Object getStructFieldData(Object object, StructField field) { + return ((OrcStruct) object).fields[((Field) field).offset]; + } + + @Override + public List getStructFieldsDataAsList(Object object) { + OrcStruct struct = (OrcStruct) object; + List result = new ArrayList(struct.fields.length); + for(Object child: struct.fields) { + result.add(child); + } + return result; + } + + @Override + public String getTypeName() { + StringBuilder buffer = new StringBuilder(); + buffer.append("struct<"); + for(int i=0; i < fields.size(); ++i) { + StructField field = fields.get(i); + if (i != 0) { + buffer.append(","); + } + buffer.append(field.getFieldName()); + buffer.append(":"); + buffer.append(field.getFieldObjectInspector().getTypeName()); + } + buffer.append(">"); + return buffer.toString(); + } + + @Override + public Category getCategory() { + return Category.STRUCT; + } + } + + static class OrcMapObjectInspector implements MapObjectInspector { + private final ObjectInspector key; + private final ObjectInspector value; + + OrcMapObjectInspector(MapTypeInfo info) { + key = createObjectInspector(info.getMapKeyTypeInfo()); + value = createObjectInspector(info.getMapValueTypeInfo()); + } + + OrcMapObjectInspector(int columnId, List types) { + OrcProto.Type type = types.get(columnId); + key = createObjectInspector(type.getSubtypes(0), types); + value = createObjectInspector(type.getSubtypes(1), types); + } + + @Override + public ObjectInspector getMapKeyObjectInspector() { + return key; + } + + @Override + public ObjectInspector getMapValueObjectInspector() { + return value; + } + + @Override + public Object getMapValueElement(Object map, Object key) { + return ((Map) map).get(key); + } + + @Override + @SuppressWarnings("unchecked") + public Map getMap(Object map) { + return (Map) map; + } + + @Override + public int getMapSize(Object map) { + return ((Map) map).size(); + } + + @Override + public String getTypeName() { + return "map<" + key.getTypeName() + "," + value.getTypeName() + ">"; + } + + @Override + public Category getCategory() { + return Category.MAP; + } + } + + static class OrcListObjectInspector implements ListObjectInspector { + private final ObjectInspector child; + + OrcListObjectInspector(ListTypeInfo info) { + child = createObjectInspector(info.getListElementTypeInfo()); + } + + OrcListObjectInspector(int columnId, List types) { + OrcProto.Type type = types.get(columnId); + child = createObjectInspector(type.getSubtypes(0), types); + } + + @Override + public ObjectInspector getListElementObjectInspector() { + return child; + } + + @Override + public Object getListElement(Object list, int i) { + return ((List) list).get(i); + } + + @Override + public int getListLength(Object list) { + return ((List) list).size(); + } + + @Override + @SuppressWarnings("unchecked") + public List getList(Object list) { + return (List) list; + } + + @Override + public String getTypeName() { + return "list<" + child.getTypeName() + ">"; + } + + @Override + public Category getCategory() { + return Category.LIST; + } + } + + static ObjectInspector createObjectInspector(TypeInfo info) { + switch (info.getCategory()) { + case PRIMITIVE: + switch (((PrimitiveTypeInfo) info).getPrimitiveCategory()) { + case FLOAT: + return PrimitiveObjectInspectorFactory.writableFloatObjectInspector; + case DOUBLE: + return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector; + case BOOLEAN: + return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector; + case BYTE: + return PrimitiveObjectInspectorFactory.writableByteObjectInspector; + case SHORT: + return PrimitiveObjectInspectorFactory.writableShortObjectInspector; + case INT: + return PrimitiveObjectInspectorFactory.writableIntObjectInspector; + case LONG: + return PrimitiveObjectInspectorFactory.writableLongObjectInspector; + case BINARY: + return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector; + case STRING: + return PrimitiveObjectInspectorFactory.writableStringObjectInspector; + case TIMESTAMP: + return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector; + default: + throw new IllegalArgumentException("Unknown primitive type " + + ((PrimitiveTypeInfo) info).getPrimitiveCategory()); + } + case STRUCT: + return new OrcStructInspector((StructTypeInfo) info); + case UNION: + return new OrcUnion.OrcUnionObjectInspector((UnionTypeInfo) info); + case MAP: + return new OrcMapObjectInspector((MapTypeInfo) info); + case LIST: + return new OrcListObjectInspector((ListTypeInfo) info); + default: + throw new IllegalArgumentException("Unknown type " + + info.getCategory()); + } + } + + static ObjectInspector createObjectInspector(int columnId, + List types){ + OrcProto.Type type = types.get(columnId); + switch (type.getKind()) { + case FLOAT: + return PrimitiveObjectInspectorFactory.writableFloatObjectInspector; + case DOUBLE: + return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector; + case BOOLEAN: + return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector; + case BYTE: + return PrimitiveObjectInspectorFactory.writableByteObjectInspector; + case SHORT: + return PrimitiveObjectInspectorFactory.writableShortObjectInspector; + case INT: + return PrimitiveObjectInspectorFactory.writableIntObjectInspector; + case LONG: + return PrimitiveObjectInspectorFactory.writableLongObjectInspector; + case BINARY: + return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector; + case STRING: + return PrimitiveObjectInspectorFactory.writableStringObjectInspector; + case TIMESTAMP: + return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector; + case STRUCT: + return new OrcStructInspector(columnId, types); + case UNION: + return new OrcUnion.OrcUnionObjectInspector(columnId, types); + case MAP: + return new OrcMapObjectInspector(columnId, types); + case LIST: + return new OrcListObjectInspector(columnId, types); + default: + throw new UnsupportedOperationException("Unknown type " + + type.getKind()); + } + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java (working copy) @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; + +class ZlibCodec implements CompressionCodec { + + @Override + public boolean compress(ByteBuffer in, ByteBuffer out, + ByteBuffer overflow) throws IOException { + Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true); + int length = in.remaining(); + deflater.setInput(in.array(), in.arrayOffset() + in.position(), length); + deflater.finish(); + int outSize = 0; + int offset = out.arrayOffset() + out.position(); + while (!deflater.finished() && (length > outSize)) { + int size = deflater.deflate(out.array(), offset, out.remaining()); + out.position(size + out.position()); + outSize += size; + offset += size; + // if we run out of space in the out buffer, use the overflow + if (out.remaining() == 0) { + if (overflow == null) { + deflater.end(); + return false; + } + out = overflow; + offset = out.arrayOffset() + out.position(); + } + } + deflater.end(); + return length > outSize; + } + + @Override + public void decompress(ByteBuffer in, ByteBuffer out) throws IOException { + Inflater inflater = new Inflater(true); + inflater.setInput(in.array(), in.arrayOffset() + in.position(), + in.remaining()); + while (!(inflater.finished() || inflater.needsDictionary() || + inflater.needsInput())) { + try { + int count = inflater.inflate(out.array(), + out.arrayOffset() + out.position(), + out.remaining()); + out.position(count + out.position()); + } catch (DataFormatException dfe) { + throw new IOException("Bad compression data", dfe); + } + } + out.flip(); + inflater.end(); + in.position(in.limit()); + } + +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/DoubleColumnStatistics.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/DoubleColumnStatistics.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/DoubleColumnStatistics.java (working copy) @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +public interface DoubleColumnStatistics extends ColumnStatistics { + + /** + * Get the smallest value in the column. Only defined if getNumberOfValues + * is non-zero. + * @return the minimum + */ + public double getMinimum(); + + /** + * Get the largest value in the column. Only defined if getNumberOfValues + * is non-zero. + * @return the maximum + */ + public double getMaximum(); + + /** + * Get the sum of the values in the column. + * @return the sum + */ + public double getSum(); +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java (working copy) @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.EOFException; +import java.io.IOException; + +/** + * A reader that reads a sequence of bytes. A control byte is read before + * each run with positive values 0 to 127 meaning 3 to 130 repetitions. If the + * byte is -1 to -128, 1 to 128 literal byte values follow. + */ +class RunLengthByteReader { + private final InStream input; + private final byte[] literals = + new byte[RunLengthByteWriter.MAX_LITERAL_SIZE]; + private int numLiterals = 0; + private int used = 0; + private boolean repeat = false; + private boolean done = false; + + RunLengthByteReader(InStream input) throws IOException { + this.input = input; + readValues(); + } + + private void readValues() throws IOException { + int control = input.read(); + used = 0; + if (control == -1) { + done = true; + } else if (control < 0x80) { + repeat = true; + numLiterals = control + RunLengthByteWriter.MIN_REPEAT_SIZE; + int val = input.read(); + if (val == -1) { + throw new EOFException("Reading RLE byte got EOF"); + } + literals[0] = (byte) val; + } else { + repeat = false; + numLiterals = 0x100 - control; + int bytes = 0; + while (bytes < numLiterals) { + int result = input.read(literals, bytes, numLiterals - bytes); + if (result == -1) { + throw new EOFException("Reading RLE byte literal got EOF"); + } + bytes += result; + } + } + } + + boolean hasNext() { + return !done && used != numLiterals; + } + + byte next() throws IOException { + byte result; + if (repeat) { + used += 1; + result = literals[0]; + } else { + result = literals[used++]; + } + if (used == numLiterals) { + readValues(); + } + return result; + } + +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java (working copy) @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +import java.io.IOException; + +public class OrcFile { + + public static final String MAGIC = "ORC"; + public static final String COMPRESSION = "orc.compress"; + static final String DEFAULT_COMPRESSION = "ZLIB"; + public static final String COMPRESSION_BLOCK_SIZE = "orc.compress.size"; + static final String DEFAULT_COMPRESSION_BLOCK_SIZE = "262144"; + public static final String STRIPE_SIZE = "orc.stripe.size"; + static final String DEFAULT_STRIPE_SIZE = "268435456"; + public static final String ROW_INDEX_STRIDE = "orc.row.index.stride"; + static final String DEFAULT_ROW_INDEX_STRIDE = "10000"; + public static final String ENABLE_INDEXES = "orc.create.index"; + + /** + * Create an ORC file reader. + * @param fs file system + * @param path file name to read from + * @return a new ORC file reader. + * @throws IOException + */ + public static Reader createReader(FileSystem fs, Path path + ) throws IOException { + return new ReaderImpl(fs, path); + } + + /** + * Create an ORC file streamFactory. + * @param fs file system + * @param path filename to write to + * @param inspector the ObjectInspector that inspects the rows + * @param stripeSize the number of bytes in a stripe + * @param compress how to compress the file + * @param bufferSize the number of bytes to compress at once + * @param rowIndexStride the number of rows between row index entries or + * 0 to suppress all indexes + * @return a new ORC file streamFactory + * @throws IOException + */ + public static Writer createWriter(FileSystem fs, + Path path, + ObjectInspector inspector, + long stripeSize, + CompressionKind compress, + int bufferSize, + int rowIndexStride) throws IOException { + return new WriterImpl(fs, path, inspector, stripeSize, compress, + bufferSize, rowIndexStride); + } + +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java (working copy) @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; + +/** + * A streamFactory that writes a sequence of bytes. A control byte is written before + * each run with positive values 0 to 127 meaning 2 to 129 repetitions. If the + * bytes is -1 to -128, 1 to 128 literal byte values follow. + */ +class RunLengthByteWriter { + static final int MIN_REPEAT_SIZE = 3; + static final int MAX_LITERAL_SIZE = 128; + static final int MAX_REPEAT_SIZE= 127 + MIN_REPEAT_SIZE; + private final PositionedOutputStream output; + private final byte[] literals = new byte[MAX_LITERAL_SIZE]; + private int numLiterals = 0; + private boolean repeat = false; + private int tailRunLength = 0; + + RunLengthByteWriter(PositionedOutputStream output) { + this.output = output; + } + + private void writeValues() throws IOException { + if (numLiterals != 0) { + if (repeat) { + output.write(numLiterals - MIN_REPEAT_SIZE); + output.write(literals, 0, 1); + } else { + output.write(-numLiterals); + output.write(literals, 0, numLiterals); + } + repeat = false; + tailRunLength = 0; + numLiterals = 0; + } + } + + void flush() throws IOException { + writeValues(); + output.flush(); + } + + void write(byte value) throws IOException { + if (numLiterals == 0) { + literals[numLiterals++] = value; + tailRunLength = 1; + } else if (repeat) { + if (value == literals[0]) { + numLiterals += 1; + if (numLiterals == MAX_REPEAT_SIZE) { + writeValues(); + } + } else { + writeValues(); + literals[numLiterals++] = value; + tailRunLength = 1; + } + } else { + if (value == literals[numLiterals - 1]) { + tailRunLength += 1; + } else { + tailRunLength = 1; + } + if (tailRunLength == MIN_REPEAT_SIZE) { + if (numLiterals + 1 == MIN_REPEAT_SIZE) { + repeat = true; + numLiterals += 1; + } else { + numLiterals -= MIN_REPEAT_SIZE - 1; + writeValues(); + literals[0] = value; + repeat = true; + numLiterals = MIN_REPEAT_SIZE; + } + } else { + literals[numLiterals++] = value; + if (numLiterals == MAX_LITERAL_SIZE) { + writeValues(); + } + } + } + } + + void getPosition(PositionRecorder recorder) throws IOException { + output.getPosition(recorder); + recorder.addPosition(numLiterals); + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java (working copy) @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +/** + * The name of a stream within a stripe. + */ +class StreamName implements Comparable { + private final int column; + private final OrcProto.Stream.Kind kind; + + public static enum Area { + DATA, INDEX + } + + public StreamName(int column, OrcProto.Stream.Kind kind) { + this.column = column; + this.kind = kind; + } + + public boolean equals(Object obj) { + if (obj != null && obj instanceof StreamName) { + StreamName other = (StreamName) obj; + return other.column == column && other.kind == kind; + } else { + return false; + } + } + + @Override + public int compareTo(StreamName streamName) { + if (streamName == null) { + return -1; + } + Area area = getArea(kind); + Area otherArea = streamName.getArea(streamName.kind); + if (area != otherArea) { + return -area.compareTo(otherArea); + } + if (column != streamName.column) { + return column < streamName.column ? -1 : 1; + } + return kind.compareTo(streamName.kind); + } + + public int getColumn() { + return column; + } + + public OrcProto.Stream.Kind getKind() { + return kind; + } + + public Area getArea() { + return getArea(kind); + } + + public static Area getArea(OrcProto.Stream.Kind kind) { + switch (kind) { + case ROW_INDEX: + case DICTIONARY_COUNT: + return Area.INDEX; + default: + return Area.DATA; + } + } + + @Override + public String toString() { + return "Stream for column " + column + " kind " + kind; + } + + @Override + public int hashCode() { + return column * 101 + kind.getNumber(); + } +} + Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java (working copy) @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; + +import java.util.ArrayList; +import java.util.List; + +/** + * An in-memory representation of a union type. + */ +final class OrcUnion implements UnionObject { + private byte tag; + private Object object; + + void set(byte tag, Object object) { + this.tag = tag; + this.object = object; + } + + @Override + public byte getTag() { + return tag; + } + + @Override + public Object getObject() { + return object; + } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != OrcUnion.class) { + return false; + } + OrcUnion oth = (OrcUnion) other; + if (tag != oth.tag) { + return false; + } else if (object == null) { + return oth.object == null; + } else { + return object.equals(oth.object); + } + } + + @Override + public int hashCode() { + int result = tag; + if (object != null) { + result ^= object.hashCode(); + } + return result; + } + + @Override + public String toString() { + return "union(" + Integer.toString(tag & 0xff) + ", " + object.toString() + + ")"; + } + + static class OrcUnionObjectInspector implements UnionObjectInspector { + private final List children; + + OrcUnionObjectInspector(int columnId, + List types) { + OrcProto.Type type = types.get(columnId); + children = new ArrayList(type.getSubtypesCount()); + for(int i=0; i < type.getSubtypesCount(); ++i) { + children.add(OrcStruct.createObjectInspector(type.getSubtypes(i), + types)); + } + } + + OrcUnionObjectInspector(UnionTypeInfo info) { + List children = info.getAllUnionObjectTypeInfos(); + this.children = new ArrayList(children.size()); + for(TypeInfo child: info.getAllUnionObjectTypeInfos()) { + this.children.add(OrcStruct.createObjectInspector(child)); + } + } + + @Override + public List getObjectInspectors() { + return children; + } + + @Override + public byte getTag(Object obj) { + return ((OrcUnion) obj).tag; + } + + @Override + public Object getField(Object obj) { + return ((OrcUnion) obj).object; + } + + @Override + public String getTypeName() { + StringBuilder builder = new StringBuilder("union{"); + boolean first = true; + for(ObjectInspector child: children) { + if (first) { + first = false; + } else { + builder.append(", "); + } + builder.append(child.getTypeName()); + } + builder.append("}"); + return builder.toString(); + } + + @Override + public Category getCategory() { + return Category.UNION; + } + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java (working copy) @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +/** + * A tool for printing out the file structure of ORC files. + */ +public class FileDump { + public static void main(String[] args) throws Exception { + Configuration conf = new Configuration(); + for(String filename: args) { + System.out.println("Structure for " + filename); + Path path = new Path(filename); + Reader reader = OrcFile.createReader(path.getFileSystem(conf), path); + RecordReaderImpl rows = (RecordReaderImpl) reader.rows(null); + System.out.println("Rows: " + reader.getNumberOfRows()); + System.out.println("Compression: " + reader.getCompression()); + if (reader.getCompression() != CompressionKind.NONE) { + System.out.println("Compression size: " + reader.getCompressionSize()); + } + System.out.println("Type: " + reader.getObjectInspector().getTypeName()); + ColumnStatistics[] stats = reader.getStatistics(); + System.out.println("\nStatistics:"); + for(int i=0; i < stats.length; ++i) { + System.out.println(" Column " + i + ": " + stats[i].toString()); + } + System.out.println("\nStripes:"); + for(StripeInformation stripe: reader.getStripes()) { + long stripeStart = stripe.getOffset(); + System.out.println(" Stripe: " + stripe.toString()); + OrcProto.StripeFooter footer = rows.readerStripeFooter(stripe); + long sectionStart = stripeStart; + for(OrcProto.Stream section: footer.getStreamsList()) { + System.out.println(" Stream: column " + section.getColumn() + + " section " + section.getKind() + " start: " + sectionStart + + " length " + section.getLength()); + sectionStart += section.getLength(); + } + } + } + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java (working copy) @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; + +class BitFieldReader { + private RunLengthByteReader input; + private final int bitSize; + private int current = 0; + private int bitsLeft = 0; + + BitFieldReader(InStream input, + int bitSize) throws IOException { + this.input = new RunLengthByteReader(input); + this.bitSize = bitSize; + } + + private void readByte() throws IOException { + if (input.hasNext()) { + current = 0xff & input.next(); + bitsLeft = 8; + } + } + + int next() throws IOException { + int result = 0; + int bitsLeftToRead = bitSize; + while (bitsLeftToRead > bitsLeft) { + result <<= bitsLeft; + result |= current >> (8 - bitsLeft); + bitsLeftToRead -= bitsLeft; + readByte(); + } + if (bitsLeftToRead > 0) { + result <<= bitsLeftToRead; + bitsLeft -= bitsLeftToRead; + result |= current >>> bitsLeft; + current &= (1 << bitsLeft) - 1; + } + return result; + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java (working copy) @@ -0,0 +1,246 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.io.Text; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * A class that is a growable array of bytes. Growth is managed in terms of + * chunks that are allocated when needed. + */ +class DynamicByteArray { + final static int DEFAULT_CHUNKSIZE = 32 * 1024; + final static int DEFAULT_NUM_CHUNKS = 128; + + final int chunkSize; /** our allocation sizes */ + byte[][] data; /** the real data */ + int length; /** max set element index +1 */ + int initializedChunks = 0; + + public DynamicByteArray() { + this(DEFAULT_NUM_CHUNKS, DEFAULT_CHUNKSIZE); + } + + public DynamicByteArray(int numChunks, int chunkSize) { + this.chunkSize = chunkSize; + + data = new byte[numChunks][]; + } + + /** + * Ensure that the given index is valid. + */ + private void grow(int chunkIndex) { + if (chunkIndex >= initializedChunks) { + if (chunkIndex >= data.length) { + int new_size = Math.max(chunkIndex, 2 * data.length); + byte[][] newChunk = new byte[new_size][]; + System.arraycopy(data, 0, newChunk, 0, data.length); + data = newChunk; + } + for(int i=initializedChunks; i <= chunkIndex; ++i) { + data[i] = new byte[chunkSize]; + } + initializedChunks = chunkIndex + 1; + } + } + + public byte get (int index) { + if (index >= length) { + throw new IndexOutOfBoundsException("Index " + index + + " is outside of 0.." + + (length - 1)); + } + int i = index / chunkSize; + int j = index % chunkSize; + return data[i][j]; + } + + public void set (int index, byte value) { + int i = index / chunkSize; + int j = index % chunkSize; + grow(i); + if (index >= length) { + length = index + 1; + } + data[i][j] = value; + } + + public int add(byte value) { + int i = length / chunkSize; + int j = length % chunkSize; + grow(i); + data[i][j] = value; + int result = length; + length += 1; + return result; + } + + public int add(byte[] value, int offset, int newLength) { + int i = length / chunkSize; + int j = length % chunkSize; + grow((length + newLength) / chunkSize); + int remaining = newLength; + while (remaining > 0) { + int size = Math.min(remaining, chunkSize - j); + System.arraycopy(value, offset, data[i], j, size); + remaining -= chunkSize - size; + i += 1; + j = 0; + } + int result = length; + length += newLength; + return result; + } + + /** + * Read the entire stream into this array + * @param in the stream to read from + * @throws IOException + */ + public void readAll(InputStream in) throws IOException { + int currentChunk = length / chunkSize; + int currentOffset = length % chunkSize; + grow(currentChunk); + int currentLength = in.read(data[currentChunk], currentOffset, + chunkSize - currentOffset); + while (currentLength > 0) { + length += currentLength; + currentOffset = length % chunkSize; + if (currentOffset == 0) { + currentChunk = length / chunkSize; + grow(currentChunk); + } + currentLength = in.read(data[currentChunk], currentOffset, + chunkSize - currentOffset); + } + } + + /** + * Byte compare a set of bytes against the bytes in this dynamic array + * @param other source of the other bytes + * @param otherOffset start offset in the other array + * @param otherLength number of bytes in the other array + * @param ourOffset the offset in our array + * @param ourLength the number of bytes in our array + * @return + */ + public int compare(byte[] other, int otherOffset, int otherLength, + int ourOffset, int ourLength) { + int currentChunk = ourOffset / chunkSize; + int currentOffset = ourOffset % chunkSize; + int maxLength = Math.min(otherLength, ourLength); + while (maxLength > 0 && + other[otherOffset] == data[currentChunk][currentOffset]) { + otherOffset += 1; + currentOffset += 1; + if (currentOffset == chunkSize) { + currentChunk += 1; + currentOffset = 0; + } + maxLength -= 1; + } + if (maxLength == 0) { + return otherLength - ourLength; + } + int otherByte = 0xff & other[otherOffset]; + int ourByte = 0xff & data[currentChunk][currentOffset]; + return otherByte > ourByte ? 1 : -1; + } + + /** + * Get the size of the array + * @return the number of bytes in the array + */ + public int size() { + return length; + } + + /** + * Clear the array to its original pristine state. + */ + public void clear() { + length = 0; + for(int i=0; i < data.length; ++i) { + data[i] = null; + } + initializedChunks = 0; + } + + /** + * Set a text value from the bytes in this dynamic array. + * @param result the value to set + * @param offset the start of the bytes to copy + * @param length the number of bytes to copy + */ + public void setText(Text result, int offset, int length) { + result.clear(); + int currentChunk = offset / chunkSize; + int currentOffset = offset % chunkSize; + int currentLength = Math.min(length, chunkSize - currentOffset); + while (length > 0) { + result.append(data[currentChunk], currentOffset, currentLength); + length -= currentLength; + currentChunk += 1; + currentOffset = 0; + currentLength = Math.min(length, chunkSize - currentOffset); + } + } + + /** + * Write out a range of this dynamic array to an output stream. + * @param out the stream to write to + * @param offset the first offset to write + * @param length the number of bytes to write + * @throws IOException + */ + public void write(OutputStream out, int offset, + int length) throws IOException { + int currentChunk = offset / chunkSize; + int currentOffset = offset % chunkSize; + while (length > 0) { + int currentLength = Math.min(length, chunkSize - currentOffset); + out.write(data[currentChunk], currentOffset, currentLength); + length -= currentLength; + currentChunk += 1; + currentOffset = 0; + } + } + + public String toString() { + int i; + StringBuilder sb = new StringBuilder(length*3); + + sb.append('{'); + int l = length-1; + for (i=0; i bitsLeft) { + int bitsToWrite = bitSize; + while (bitsToWrite > 0) { + // add the bits to the bottom of the current word + current |= value >>> (bitsToWrite - bitsLeft); + // subtract out the bits we just added + bitsToWrite -= bitsLeft; + // zero out the bits above bitsToWrite + value &= (1 << bitsToWrite) - 1; + writeByte(); + } + } else { + bitsLeft -= bitSize; + current |= value << bitsLeft; + if (bitsLeft == 0) { + writeByte(); + } + } + } + + void getPosition(PositionRecorder recorder) throws IOException { + output.getPosition(recorder); + recorder.addPosition(8-bitsLeft); + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (working copy) @@ -0,0 +1,973 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.io.ByteWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class RecordReaderImpl implements RecordReader { + private final FSDataInputStream file; + private final long firstRow; + private final List stripes = + new ArrayList(); + private final long totalRowCount; + private final CompressionCodec codec; + private final int bufferSize; + private final boolean[] included; + private long rowInStripe = 0; + private int currentStripe = 0; + private long rowBaseInStripe = 0; + private long rowCountInStripe = 0; + private final Map streams = + new HashMap(); + private final TreeReader reader; + + RecordReaderImpl(Iterable stripes, + FileSystem fileSystem, + Path path, + long offset, long length, + List types, + CompressionCodec codec, + int bufferSize, + boolean[] included + ) throws IOException { + this.file = fileSystem.open(path); + this.codec = codec; + this.bufferSize = bufferSize; + this.included = included; + long rows = 0; + long skippedRows = 0; + for(StripeInformation stripe: stripes) { + long stripeStart = stripe.getOffset(); + if (offset > stripeStart) { + skippedRows += stripe.getNumberOfRows(); + } else if (stripeStart < offset+length) { + this.stripes.add(stripe); + rows += stripe.getNumberOfRows(); + } + } + firstRow = skippedRows; + totalRowCount = rows; + reader = createTreeReader(0, types, included); + if (this.stripes.size() > 0) { + readStripe(); + } + } + + private abstract static class TreeReader { + protected final int columnId; + private BitFieldReader present = null; + protected boolean valuePresent = false; + + TreeReader(int columnId) { + this.columnId = columnId; + } + + void startStripe(Map streams + ) throws IOException { + InStream in = streams.get(new StreamName(columnId, + OrcProto.Stream.Kind.PRESENT)); + if (in == null) { + present = null; + valuePresent = true; + } else { + present = new BitFieldReader(in, 1); + } + } + + abstract void seekToRow(long row) throws IOException; + + Object next(Object previous) throws IOException { + if (present != null) { + valuePresent = present.next() == 1; + } + return previous; + } + } + + private static class BooleanTreeReader extends TreeReader{ + private BitFieldReader reader = null; + + BooleanTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + reader = new BitFieldReader(streams.get(new StreamName + (columnId, OrcProto.Stream.Kind.DATA)), 1); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + BooleanWritable result = null; + if (valuePresent) { + if (previous == null) { + result = new BooleanWritable(); + } else { + result = (BooleanWritable) previous; + } + result.set(reader.next() == 1); + } + return result; + } + } + + private static class ByteTreeReader extends TreeReader{ + private RunLengthByteReader reader = null; + + ByteTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + reader = new RunLengthByteReader(streams.get(new StreamName + (columnId, OrcProto.Stream.Kind.DATA))); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + ByteWritable result = null; + if (valuePresent) { + if (previous == null) { + result = new ByteWritable(); + } else { + result = (ByteWritable) previous; + } + result.set(reader.next()); + } + return result; + } + } + + private static class ShortTreeReader extends TreeReader{ + private RunLengthIntegerReader reader = null; + + ShortTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + StreamName name = new StreamName(columnId, + OrcProto.Stream.Kind.DATA); + reader = new RunLengthIntegerReader(streams.get(name), true); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + ShortWritable result = null; + if (valuePresent) { + if (previous == null) { + result = new ShortWritable(); + } else { + result = (ShortWritable) previous; + } + result.set((short) reader.next()); + } + return result; + } + } + + private static class IntTreeReader extends TreeReader{ + private RunLengthIntegerReader reader = null; + + IntTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + StreamName name = new StreamName(columnId, + OrcProto.Stream.Kind.DATA); + reader = new RunLengthIntegerReader(streams.get(name), true); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + IntWritable result = null; + if (valuePresent) { + if (previous == null) { + result = new IntWritable(); + } else { + result = (IntWritable) previous; + } + result.set((int) reader.next()); + } + return result; + } + } + + private static class LongTreeReader extends TreeReader{ + private RunLengthIntegerReader reader = null; + + LongTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + StreamName name = new StreamName(columnId, + OrcProto.Stream.Kind.DATA); + reader = new RunLengthIntegerReader(streams.get(name), true); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + LongWritable result = null; + if (valuePresent) { + if (previous == null) { + result = new LongWritable(); + } else { + result = (LongWritable) previous; + } + result.set(reader.next()); + } + return result; + } + } + + private static class FloatTreeReader extends TreeReader{ + private InStream stream; + + FloatTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + StreamName name = new StreamName(columnId, + OrcProto.Stream.Kind.DATA); + stream = streams.get(name); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + FloatWritable result = null; + if (valuePresent) { + if (previous == null) { + result = new FloatWritable(); + } else { + result = (FloatWritable) previous; + } + result.set(SerializationUtils.readFloat(stream)); + } + return result; + } + } + + private static class DoubleTreeReader extends TreeReader{ + private InStream stream; + + DoubleTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams) throws IOException { + super.startStripe(streams); + StreamName name = + new StreamName(columnId, + OrcProto.Stream.Kind.DATA); + stream = streams.get(name); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + DoubleWritable result = null; + if (valuePresent) { + if (previous == null) { + result = new DoubleWritable(); + } else { + result = (DoubleWritable) previous; + } + result.set(SerializationUtils.readDouble(stream)); + } + return result; + } + } + + private static class BinaryTreeReader extends TreeReader{ + private InStream stream; + private RunLengthIntegerReader lengths; + + BinaryTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + StreamName name = new StreamName(columnId, + OrcProto.Stream.Kind.DATA); + stream = streams.get(name); + lengths = new RunLengthIntegerReader(streams.get(new + StreamName(columnId, OrcProto.Stream.Kind.LENGTH)), + false); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + BytesWritable result = null; + if (valuePresent) { + if (previous == null) { + result = new BytesWritable(); + } else { + result = (BytesWritable) previous; + } + int len = (int) lengths.next(); + result.setSize(len); + int offset = 0; + while (len > 0) { + int written = stream.read(result.getBytes(), offset, len); + if (written < 0) { + throw new EOFException("Can't finish byte read from " + stream); + } + len -= written; + offset += written; + } + } + return result; + } + } + + private static class TimestampTreeReader extends TreeReader{ + private RunLengthIntegerReader data; + private RunLengthIntegerReader nanos; + + TimestampTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams) throws IOException { + super.startStripe(streams); + data = new RunLengthIntegerReader(streams.get(new StreamName + (columnId, OrcProto.Stream.Kind.DATA)), true); + nanos = new RunLengthIntegerReader(streams.get(new StreamName + (columnId, OrcProto.Stream.Kind.NANO_DATA)), false); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + Timestamp result = null; + if (valuePresent) { + if (previous == null) { + result = new Timestamp(0); + } else { + result = (Timestamp) previous; + } + long millis = (data.next() + WriterImpl.BASE_TIMESTAMP) * + WriterImpl.MILLIS_PER_SECOND; + int nanos = parseNanos(this.nanos.next()); + // fix the rounding when we divided by 1000. + if (millis >= 0) { + millis += nanos / 1000000; + } else { + millis -= nanos / 1000000; + } + result.setTime(millis); + result.setNanos(nanos); + } + return result; + } + + private static int parseNanos(long serialized) { + int zeros = 7 & (int) serialized; + int result = (int) serialized >>> 3; + if (zeros != 0) { + for(int i =0; i <= zeros; ++i) { + result *= 10; + } + } + return result; + } + } + + private static class StringTreeReader extends TreeReader { + private DynamicByteArray dictionaryBuffer = null; + private final DynamicIntArray dictionaryOffsets = new DynamicIntArray(); + private final DynamicIntArray dictionaryLengths = new DynamicIntArray(); + private RunLengthIntegerReader reader; + + StringTreeReader(int columnId) { + super(columnId); + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + + // read the dictionary blob + StreamName name = new StreamName(columnId, + OrcProto.Stream.Kind.DICTIONARY_DATA); + InStream in = streams.get(name); + dictionaryBuffer = new DynamicByteArray(64, in.available()); + dictionaryBuffer.readAll(in); + in.close(); + + // read the lengths + name = new StreamName(columnId, + OrcProto.Stream.Kind.LENGTH); + in = streams.get(name); + RunLengthIntegerReader lenReader = new RunLengthIntegerReader(in, false); + int offset = 0; + dictionaryOffsets.clear(); + dictionaryLengths.clear(); + while (lenReader.hasNext()) { + dictionaryOffsets.add(offset); + int len = (int) lenReader.next(); + dictionaryLengths.add(len); + offset += len; + } + in.close(); + + // set up the row reader + name = new StreamName(columnId, OrcProto.Stream.Kind.DATA); + reader = new RunLengthIntegerReader(streams.get(name), false); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + Text result = null; + if (valuePresent) { + int entry = (int) reader.next(); + if (previous == null) { + result = new Text(); + } else { + result = (Text) previous; + } + int offset = dictionaryOffsets.get(entry); + int length = dictionaryLengths.get(entry); + dictionaryBuffer.setText(result, offset, length); + } + return result; + } + } + + private static class StructTreeReader extends TreeReader { + private final TreeReader[] fields; + private final String[] fieldNames; + + StructTreeReader(int columnId, + List types, + boolean[] included) throws IOException { + super(columnId); + OrcProto.Type type = types.get(columnId); + int fieldCount = type.getFieldNamesCount(); + this.fields = new TreeReader[fieldCount]; + this.fieldNames = new String[fieldCount]; + for(int i=0; i < fieldCount; ++i) { + int subtype = type.getSubtypes(i); + if (included == null || included[subtype]) { + this.fields[i] = createTreeReader(subtype, types, included); + } + this.fieldNames[i] = type.getFieldNames(i); + } + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + OrcStruct result = null; + if (valuePresent) { + if (previous == null) { + result = new OrcStruct(fields.length); + } else { + result = (OrcStruct) previous; + } + for(int i=0; i < fields.length; ++i) { + if (fields[i] != null) { + result.setFieldValue(i, fields[i].next(result.getFieldValue(i))); + } + } + } + return result; + } + + @Override + void startStripe(Map streams) throws IOException { + super.startStripe(streams); + for(TreeReader field: fields) { + if (field != null) { + field.startStripe(streams); + } + } + } + } + + private static class UnionTreeReader extends TreeReader { + private final TreeReader[] fields; + private RunLengthByteReader tags; + + UnionTreeReader(int columnId, + List types, + boolean[] included) throws IOException { + super(columnId); + OrcProto.Type type = types.get(columnId); + int fieldCount = type.getSubtypesCount(); + this.fields = new TreeReader[fieldCount]; + for(int i=0; i < fieldCount; ++i) { + int subtype = type.getSubtypes(i); + if (included == null || included[subtype]) { + this.fields[i] = createTreeReader(subtype, types, included); + } + } + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + Object next(Object previous) throws IOException { + super.next(previous); + OrcUnion result = null; + if (valuePresent) { + if (previous == null) { + result = new OrcUnion(); + } else { + result = (OrcUnion) previous; + } + byte tag = (byte) tags.next(); + Object previousVal = result.getObject(); + result.set(tag, fields[tag].next(tag == result.getTag() ? + previousVal : null)); + } + return result; + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + tags = new RunLengthByteReader(streams.get(new StreamName + (columnId, OrcProto.Stream.Kind.DATA))); + for(TreeReader field: fields) { + if (field != null) { + field.startStripe(streams); + } + } + } + } + + private static class ListTreeReader extends TreeReader { + private final TreeReader elementReader; + private RunLengthIntegerReader lengths; + + ListTreeReader(int columnId, + List types, + boolean[] included) throws IOException { + super(columnId); + OrcProto.Type type = types.get(columnId); + elementReader = createTreeReader(type.getSubtypes(0), types, included); + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + @SuppressWarnings("unchecked") + Object next(Object previous) throws IOException { + super.next(previous); + ArrayList result = null; + if (valuePresent) { + if (previous == null) { + result = new ArrayList(); + } else { + result = (ArrayList) previous; + } + int prevLength = result.size(); + int length = (int) lengths.next(); + // extend the list to the new length + for(int i=prevLength; i < length; ++i) { + result.add(null); + } + // read the new elements into the array + for(int i=0; i< length; i++) { + result.set(i, elementReader.next(i < prevLength ? + result.get(i) : null)); + } + // remove any extra elements + for(int i=prevLength - 1; i >= length; --i) { + result.remove(i); + } + } + return result; + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + lengths = new RunLengthIntegerReader(streams.get(new StreamName + (columnId, OrcProto.Stream.Kind.LENGTH)), false); + if (elementReader != null) { + elementReader.startStripe(streams); + } + } + } + + private static class MapTreeReader extends TreeReader { + private final TreeReader keyReader; + private final TreeReader valueReader; + private RunLengthIntegerReader lengths; + + MapTreeReader(int columnId, + List types, + boolean[] included) throws IOException { + super(columnId); + OrcProto.Type type = types.get(columnId); + int keyColumn = type.getSubtypes(0); + int valueColumn = type.getSubtypes(1); + if (included == null || included[keyColumn]) { + keyReader = createTreeReader(keyColumn, types, included); + } else { + keyReader = null; + } + if (included == null || included[valueColumn]) { + valueReader = createTreeReader(valueColumn, types, included); + } else { + valueReader = null; + } + } + + @Override + void seekToRow(long row) throws IOException { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + @SuppressWarnings("unchecked") + Object next(Object previous) throws IOException { + super.next(previous); + HashMap result = null; + if (valuePresent) { + if (previous == null) { + result = new HashMap(); + } else { + result = (HashMap) previous; + } + // for now just clear and create new objects + result.clear(); + int length = (int) lengths.next(); + // read the new elements into the array + for(int i=0; i< length; i++) { + result.put(keyReader.next(null), valueReader.next(null)); + } + } + return result; + } + + @Override + void startStripe(Map streams + ) throws IOException { + super.startStripe(streams); + lengths = new RunLengthIntegerReader(streams.get(new StreamName + (columnId, OrcProto.Stream.Kind.LENGTH)), false); + if (keyReader != null) { + keyReader.startStripe(streams); + } + if (valueReader != null) { + valueReader.startStripe(streams); + } + } + } + + private static TreeReader createTreeReader(int columnId, + List types, + boolean[] included + ) throws IOException { + OrcProto.Type type = types.get(columnId); + switch (type.getKind()) { + case BOOLEAN: + return new BooleanTreeReader(columnId); + case BYTE: + return new ByteTreeReader(columnId); + case DOUBLE: + return new DoubleTreeReader(columnId); + case FLOAT: + return new FloatTreeReader(columnId); + case SHORT: + return new ShortTreeReader(columnId); + case INT: + return new IntTreeReader(columnId); + case LONG: + return new LongTreeReader(columnId); + case STRING: + return new StringTreeReader(columnId); + case BINARY: + return new BinaryTreeReader(columnId); + case TIMESTAMP: + return new TimestampTreeReader(columnId); + case STRUCT: + return new StructTreeReader(columnId, types, included); + case LIST: + return new ListTreeReader(columnId, types, included); + case MAP: + return new MapTreeReader(columnId, types, included); + case UNION: + return new UnionTreeReader(columnId, types, included); + default: + throw new IllegalArgumentException("Unsupported type " + + type.getKind()); + } + } + + OrcProto.StripeFooter readerStripeFooter(StripeInformation stripe + ) throws IOException { + long offset = stripe.getOffset() + stripe.getIndexLength() + + stripe.getDataLength(); + int tailLength = (int) stripe.getFooterLength(); + + // read the footer + ByteBuffer tailBuf = ByteBuffer.allocate(tailLength); + file.seek(offset); + file.readFully(tailBuf.array(), tailBuf.arrayOffset(), tailLength); + return OrcProto.StripeFooter.parseFrom(InStream.create("footer", tailBuf, + codec, bufferSize)); + } + + private boolean includeColumn(int id) { + return included[id]; + } + + private void readStripe() throws IOException { + StripeInformation stripe = stripes.get(currentStripe); + OrcProto.StripeFooter footer = readerStripeFooter(stripe); + long offset = stripe.getOffset(); + streams.clear(); + + // if we aren't projecting columns, just read the whole stripe + if (included == null) { + byte[] buffer = + new byte[(int) (stripe.getDataLength())]; + file.seek(offset + stripe.getIndexLength()); + file.readFully(buffer, 0, buffer.length); + int sectionOffset = 0; + for(OrcProto.Stream section: footer.getStreamsList()) { + if (StreamName.getArea(section.getKind()) == StreamName.Area.DATA) { + int sectionLength = (int) section.getLength(); + ByteBuffer sectionBuffer = ByteBuffer.wrap(buffer, sectionOffset, + sectionLength); + StreamName name = new StreamName(section.getColumn(), + section.getKind()); + streams.put(name, + InStream.create(name.toString(), sectionBuffer, codec, + bufferSize)); + sectionOffset += sectionLength; + } + } + } else { + List streams = footer.getStreamsList(); + // the index of the current section + int currentSection = 0; + while (currentSection < streams.size() && + StreamName.getArea(streams.get(currentSection).getKind()) != + StreamName.Area.DATA) { + currentSection += 1; + } + // byte position of the current section relative to the stripe start + long sectionOffset = stripe.getIndexLength(); + while (currentSection < streams.size()) { + int bytes = 0; + + // find the first section that shouldn't be read + int excluded=currentSection; + while (excluded < streams.size() && + includeColumn(streams.get(excluded).getColumn())) { + bytes += streams.get(excluded).getLength(); + excluded += 1; + } + + // actually read the bytes as a big chunk + if (bytes != 0) { + byte[] buffer = new byte[bytes]; + file.seek(offset + sectionOffset); + file.readFully(buffer, 0, bytes); + sectionOffset += bytes; + + // create the streams for the sections we just read + bytes = 0; + while (currentSection < excluded) { + OrcProto.Stream section = streams.get(currentSection); + StreamName name = + new StreamName(section.getColumn(), section.getKind()); + this.streams.put(name, + InStream.create(name.toString(), + ByteBuffer.wrap(buffer, bytes, + (int) section.getLength()), codec, bufferSize)); + currentSection += 1; + bytes += section.getLength(); + } + } + + // skip forward until we get back to a section that we need + while (currentSection < streams.size() && + !includeColumn(streams.get(currentSection).getColumn())) { + sectionOffset += streams.get(currentSection).getLength(); + currentSection += 1; + } + } + } + reader.startStripe(streams); + rowInStripe = 0; + rowCountInStripe = stripe.getNumberOfRows(); + rowBaseInStripe = 0; + for(int i=0; i < currentStripe; ++i) { + rowBaseInStripe += stripes.get(i).getNumberOfRows(); + } + } + + @Override + public boolean hasNext() throws IOException { + return rowInStripe < rowCountInStripe || currentStripe < stripes.size() - 1; + } + + @Override + public Object next(Object previous) throws IOException { + if (rowInStripe >= rowCountInStripe) { + currentStripe += 1; + readStripe(); + } + rowInStripe += 1; + return reader.next(previous); + } + + @Override + public void close() throws IOException { + file.close(); + } + + @Override + public long getRowNumber() { + return rowInStripe + rowBaseInStripe + firstRow; + } + + /** + * Return the fraction of rows that have been read from the selected + * section of the file + * @return fraction between 0.0 and 1.0 of rows consumed + */ + @Override + public float getProgress() { + return ((float) rowBaseInStripe + rowInStripe)/totalRowCount; + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerColumnStatistics.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerColumnStatistics.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerColumnStatistics.java (working copy) @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +public interface IntegerColumnStatistics extends ColumnStatistics { + /** + * Get the smallest value in the column. Only defined if getNumberOfValues + * is non-zero. + * @return the minimum + */ + long getMinimum(); + + /** + * Get the largest value in the column. Only defined if getNumberOfValues + * is non-zero. + * @return the maximum + */ + long getMaximum(); + + /** + * Is the sum defined? If the sum overflowed the counter this will be false. + * @return is the sum available + */ + boolean isSumDefined(); + + /** + * Get the sum of the column. Only valid if isSumDefined returns true. + * @return the sum of the column + */ + long getSum(); +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java (working copy) @@ -0,0 +1,327 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +/** + * A memory efficient red-black tree that does not allocate any objects per + * an element. This class is abstract and assumes that the child class + * handles the key and comparisons with the key. + */ +abstract class RedBlackTree +{ + public static final int NULL = -1; + private static final int DEFAULT_INITIAL_CAPACITY = 16*1024; + + // Various values controlling the offset of the data within the array. + private static final int LEFT_OFFSET = 0; + private static final int RIGHT_OFFSET = 1; + private static final int COUNT_OFFSET = 2; + private static final int ELEMENT_SIZE = 3; + + protected int size = 0; + private final DynamicIntArray data; + protected int root = NULL; + protected int lastAdd = 0; + private boolean wasAdd = false; + + /** + * Create a set with a default initial capacity. + */ + public RedBlackTree() { + data = new DynamicIntArray(DEFAULT_INITIAL_CAPACITY * ELEMENT_SIZE); + } + + /** + * Create a set with the given initial capacity. + */ + public RedBlackTree(int initial_capacity) { + data = new DynamicIntArray(initial_capacity * ELEMENT_SIZE); + } + + /** + * Insert a new node into the data array, growing the array as necessary. + * + * @return Returns the position of the new node. + */ + private int insert(int left, int right, boolean isRed) { + int position = size; + size += 1; + setLeft(position, left, isRed); + setRight(position, right); + setCount(position, 1); + return position; + } + + /** + * Compare the value at the given position to the new value. + * @return 0 if the values are the same, -1 if the new value is smaller and + * 1 if the new value is larger. + */ + protected abstract int compareValue(int position); + + /** + * Is the given node red as opposed to black? To prevent having an extra word + * in the data array, we just the low bit on the left child index. + */ + protected boolean isRed(int position) { + return position != NULL && + (data.get(position * ELEMENT_SIZE + LEFT_OFFSET) & 1) == 1; + } + + /** + * Set the red bit true or false. + */ + private void setRed(int position, boolean is_red) { + int offset = position * ELEMENT_SIZE + LEFT_OFFSET; + if (is_red) { + data.set(offset, data.get(offset) | 1); + } else { + data.set(offset, data.get(offset) & ~1); + } + } + + /** + * Get the left field of the given position. + */ + protected int getLeft(int position) { + return data.get(position * ELEMENT_SIZE + LEFT_OFFSET) >> 1; + } + + /** + * Get the right field of the given position. + */ + protected int getRight(int position) { + return data.get(position * ELEMENT_SIZE + RIGHT_OFFSET); + } + + protected int getCount(int position) { + return data.get(position * ELEMENT_SIZE + COUNT_OFFSET); + } + + private void setCount(int position, int value) { + data.set(position * ELEMENT_SIZE + COUNT_OFFSET, value); + } + + private void incrementCount(int position, int value) { + data.increment(position * ELEMENT_SIZE + COUNT_OFFSET, value); + } + + /** + * Set the left field of the given position. + * Note that we are storing the node color in the low bit of the left pointer. + */ + private void setLeft(int position, int left) { + int offset = position * ELEMENT_SIZE + LEFT_OFFSET; + data.set(offset, (left << 1) | (data.get(offset) & 1)); + } + + /** + * Set the left field of the given position. + * Note that we are storing the node color in the low bit of the left pointer. + */ + private void setLeft(int position, int left, boolean isRed) { + int offset = position * ELEMENT_SIZE + LEFT_OFFSET; + data.set(offset, (left << 1) | (isRed ? 1 : 0)); + } + + /** + * Set the right field of the given position. + */ + private void setRight(int position, int right) { + data.set(position * ELEMENT_SIZE + RIGHT_OFFSET, right); + } + + /** + * Insert or find a given key in the tree and rebalance the tree correctly. + * Rebalancing restores the red-black aspect of the tree to maintain the + * invariants: + * 1. If a node is red, both of its children are black. + * 2. Each child of a node has the same black height (the number of black + * nodes between it and the leaves of the tree). + * + * Inserted nodes are at the leaves and are red, therefore there is at most a + * violation of rule 1 at the node we just put in. Instead of always keeping + * the parents, this routine passing down the context. + * + * The fix is broken down into 6 cases (1.{1,2,3} and 2.{1,2,3} that are + * left-right mirror images of each other). See Algorighms by Cormen, + * Leiserson, and Rivest for the explaination of the subcases. + * + * @param node The node that we are fixing right now. + * @param fromLeft Did we come down from the left? + * @param parent Nodes' parent + * @param grandparent Parent's parent + * @param great_grandparent Grandparent's parent + * @return Does parent also need to be checked and/or fixed? + */ + private boolean add(int node, boolean fromLeft, int parent, + int grandparent, int great_grandparent) { + if (node == NULL) { + if (root == NULL) { + lastAdd = insert(NULL, NULL, false); + root = lastAdd; + wasAdd = true; + return false; + } else { + lastAdd = insert(NULL, NULL, true); + node = lastAdd; + wasAdd = true; + // connect the new node into the tree + if (fromLeft) { + setLeft(parent, node); + } else { + setRight(parent, node); + } + } + } else { + int compare = compareValue(node); + boolean keep_going; + + // Recurse down to find where the node needs to be added + if (compare < 0) { + keep_going = add(getLeft(node), true, node, parent, grandparent); + } else if (compare > 0) { + keep_going = add(getRight(node), false, node, parent, grandparent); + } else { + lastAdd = node; + wasAdd = false; + incrementCount(node, 1); + return false; + } + + // we don't need to fix the root (because it is always set to black) + if (node == root || !keep_going) { + return false; + } + } + + + // Do we need to fix this node? Only if there are two reds right under each + // other. + if (isRed(node) && isRed(parent)) { + if (parent == getLeft(grandparent)) { + int uncle = getRight(grandparent); + if (isRed(uncle)) { + // case 1.1 + setRed(parent, false); + setRed(uncle, false); + setRed(grandparent, true); + return true; + } else { + if (node == getRight(parent)) { + // case 1.2 + // swap node and parent + int tmp = node; + node = parent; + parent = tmp; + // left-rotate on node + setLeft(grandparent, parent); + setRight(node, getLeft(parent)); + setLeft(parent, node); + } + + // case 1.2 and 1.3 + setRed(parent, false); + setRed(grandparent, true); + + // right-rotate on grandparent + if (great_grandparent == NULL) { + root = parent; + } else if (getLeft(great_grandparent) == grandparent) { + setLeft(great_grandparent, parent); + } else { + setRight(great_grandparent, parent); + } + setLeft(grandparent, getRight(parent)); + setRight(parent, grandparent); + return false; + } + } else { + int uncle = getLeft(grandparent); + if (isRed(uncle)) { + // case 2.1 + setRed(parent, false); + setRed(uncle, false); + setRed(grandparent, true); + return true; + } else { + if (node == getLeft(parent)) { + // case 2.2 + // swap node and parent + int tmp = node; + node = parent; + parent = tmp; + // right-rotate on node + setRight(grandparent, parent); + setLeft(node, getRight(parent)); + setRight(parent, node); + } + // case 2.2 and 2.3 + setRed(parent, false); + setRed(grandparent, true); + // left-rotate on grandparent + if (great_grandparent == NULL) { + root = parent; + } else if (getRight(great_grandparent) == grandparent) { + setRight(great_grandparent, parent); + } else { + setLeft(great_grandparent, parent); + } + setRight(grandparent, getLeft(parent)); + setLeft(parent, grandparent); + return false; + } + } + } else { + return true; + } + } + + /** + * Add the new key to the tree. + * @return true if the element is a new one. + */ + protected boolean add() { + add(root, false, NULL, NULL, NULL); + if (wasAdd) { + setRed(root, false); + return true; + } else { + return false; + } + } + + /** + * Get the number of elements in the set. + */ + public int size() { + return size; + } + + /** + * Reset the table to empty. + */ + public void clear() { + root = NULL; + size = 0; + data.clear(); + } + +} + Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java (working copy) @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; + +public interface RecordReader { + /** + * Does the reader have more rows available + * @return true if there are more rows + * @throws java.io.IOException + */ + boolean hasNext() throws IOException; + + /** + * Read the next row + * @param previous a row object that can be reused by the reader + * @return the row that was read + * @throws java.io.IOException + */ + Object next(Object previous) throws IOException; + + /** + * Get the row number of the row that will be returned by the following + * call to next(). + * @return the row number from 0 to the number of rows in the file + * @throws java.io.IOException + */ + long getRowNumber() throws IOException; + + /** + * Get the progress of the reader through the rows. + * @return a fraction between 0.0 and 1.0 of rows read + * @throws java.io.IOException + */ + float getProgress() throws IOException; + + /** + * Release the resources associated with the given reader. + * @throws java.io.IOException + */ + void close() throws IOException; +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java (working copy) @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.Writable; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Properties; + +public class OrcSerde implements SerDe { + private final OrcSerdeRow row = new OrcSerdeRow(); + private ObjectInspector inspector = null; + + final class OrcSerdeRow implements Writable { + Object realRow; + ObjectInspector inspector; + + @Override + public void write(DataOutput dataOutput) throws IOException { + throw new UnsupportedOperationException("can't write the bundle"); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + throw new UnsupportedOperationException("can't read the bundle"); + } + } + + @Override + public void initialize(Configuration conf, Properties table) { + // Read the configuration parameters + String columnNameProperty = table.getProperty("columns"); + // NOTE: if "columns.types" is missing, all columns will be of String type + String columnTypeProperty = table.getProperty("columns.types"); + + // Parse the configuration parameters + ArrayList columnNames = new ArrayList(); + if (columnNameProperty != null && columnNameProperty.length() > 0) { + for(String name: columnNameProperty.split(",")) { + columnNames.add(name); + } + } + if (columnTypeProperty == null) { + // Default type: all string + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < columnNames.size(); i++) { + if (i > 0) { + sb.append(":"); + } + sb.append("string"); + } + columnTypeProperty = sb.toString(); + } + + ArrayList fieldTypes = + TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty); + StructTypeInfo rootType = new StructTypeInfo(); + rootType.setAllStructFieldNames(columnNames); + rootType.setAllStructFieldTypeInfos(fieldTypes); + inspector = OrcStruct.createObjectInspector(rootType); + } + + @Override + public Class getSerializedClass() { + return OrcSerdeRow.class; + } + + @Override + public Writable serialize(Object realRow, ObjectInspector inspector) { + row.realRow = realRow; + row.inspector = inspector; + return row; + } + + @Override + public Object deserialize(Writable writable) throws SerDeException { + return writable; + } + + @Override + public ObjectInspector getObjectInspector() throws SerDeException { + return inspector; + } + + /** + * Always returns null, since serialized size doesn't make sense in the + * context of ORC files. + * @return null + */ + @Override + public SerDeStats getSerDeStats() { + return null; + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java (working copy) @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.iq80.snappy.Snappy; + +import java.io.IOException; +import java.nio.ByteBuffer; + +class SnappyCodec implements CompressionCodec { + + @Override + public boolean compress(ByteBuffer in, ByteBuffer out, + ByteBuffer overflow) throws IOException { + int inBytes = in.remaining(); + // I should work on a patch for Snappy to support an overflow buffer + // to prevent the extra buffer copy. + byte[] compressed = new byte[Snappy.maxCompressedLength(inBytes)]; + int outBytes = + Snappy.compress(in.array(), in.arrayOffset() + in.position(), inBytes, + compressed, 0); + if (outBytes < inBytes) { + int remaining = out.remaining(); + if (remaining >= outBytes) { + System.arraycopy(compressed, 0, out.array(), out.arrayOffset() + + out.position(), outBytes); + out.position(out.position() + outBytes); + } else { + System.arraycopy(compressed, 0, out.array(), out.arrayOffset() + + out.position(), remaining); + out.position(out.limit()); + System.arraycopy(compressed, remaining, overflow.array(), + overflow.arrayOffset(), outBytes - remaining); + overflow.position(outBytes - remaining); + } + return true; + } else { + return false; + } + } + + @Override + public void decompress(ByteBuffer in, ByteBuffer out) throws IOException { + int inOffset = in.position(); + int uncompressLen = + Snappy.uncompress(in.array(), in.arrayOffset() + inOffset, + in.limit() - inOffset, out.array(), out.arrayOffset() + out.position()); + out.position(uncompressLen + out.position()); + out.flip(); + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/PositionedOutputStream.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/PositionedOutputStream.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/PositionedOutputStream.java (working copy) @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.io.OutputStream; + +abstract class PositionedOutputStream extends OutputStream { + abstract void getPosition(PositionRecorder recorder) throws IOException; +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java (working copy) @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +abstract class InStream extends InputStream { + + private static class UncompressedStream extends InStream { + private final String name; + private byte[] array; + private int offset; + private int limit; + + public UncompressedStream(String name, ByteBuffer input) { + this.name = name; + this.array = input.array(); + offset = input.arrayOffset() + input.position(); + limit = input.arrayOffset() + input.limit(); + } + + @Override + public int read() { + if (offset == limit) { + return -1; + } + return 0xff & array[offset++]; + } + + @Override + public int read(byte[] data, int offset, int length) { + if (this.offset == limit) { + return -1; + } + int actualLength = Math.min(length, limit - this.offset); + System.arraycopy(array, this.offset, data, offset, actualLength); + this.offset += actualLength; + return actualLength; + } + + @Override + public int available() { + return limit - offset; + } + + @Override + public void close() { + array = null; + offset = 0; + limit = 0; + } + } + + private static class CompressedStream extends InStream { + private final String name; + private byte[] array; + private final int bufferSize; + private ByteBuffer uncompressed = null; + private final CompressionCodec codec; + private int offset; + private int limit; + private boolean isUncompressedOriginal; + + public CompressedStream(String name, ByteBuffer input, + CompressionCodec codec, int bufferSize + ) { + this.array = input.array(); + this.name = name; + this.codec = codec; + this.bufferSize = bufferSize; + offset = input.arrayOffset() + input.position(); + limit = input.arrayOffset() + input.limit(); + } + + private void readHeader() throws IOException { + if (limit - offset > OutStream.HEADER_SIZE) { + int chunkLength = ((0xff & array[offset+2]) << 15) | + ((0xff & array[offset+1]) << 7) | ((0xff & array[offset]) >> 1); + boolean isOriginal = (array[offset] & 0x01) == 1; + offset += OutStream.HEADER_SIZE; + if (isOriginal) { + isUncompressedOriginal = true; + uncompressed = ByteBuffer.wrap(array, offset, chunkLength); + } else { + if (isUncompressedOriginal) { + uncompressed = ByteBuffer.allocate(bufferSize); + isUncompressedOriginal = false; + } else if (uncompressed == null) { + uncompressed = ByteBuffer.allocate(bufferSize); + } else { + uncompressed.clear(); + } + codec.decompress(ByteBuffer.wrap(array, offset, chunkLength), + uncompressed); + } + offset += chunkLength; + } else { + throw new IllegalStateException("Can't read header"); + } + } + + @Override + public int read() throws IOException { + if (uncompressed == null || uncompressed.remaining() == 0) { + if (offset == limit) { + return -1; + } + readHeader(); + } + return 0xff & uncompressed.get(); + } + + @Override + public int read(byte[] data, int offset, int length) throws IOException { + if (uncompressed == null || uncompressed.remaining() == 0) { + if (this.offset == this.limit) { + return -1; + } + readHeader(); + } + int actualLength = Math.min(length, uncompressed.remaining()); + System.arraycopy(uncompressed.array(), + uncompressed.arrayOffset() + uncompressed.position(), data, + offset, actualLength); + uncompressed.position(uncompressed.position() + actualLength); + return actualLength; + } + + @Override + public int available() throws IOException { + if (uncompressed == null || uncompressed.remaining() == 0) { + if (offset == limit) { + return 0; + } + readHeader(); + } + return uncompressed.remaining(); + } + + @Override + public void close() { + array = null; + uncompressed = null; + limit = 0; + offset = 0; + } + } + + public static InStream create(String name, + ByteBuffer input, + CompressionCodec codec, + int bufferSize) throws IOException { + if (codec == null) { + return new UncompressedStream(name, input); + } else { + return new CompressedStream(name, input, codec, bufferSize); + } + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java (working copy) @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import com.google.protobuf.CodedInputStream; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +class ReaderImpl implements Reader { + + private static final int DIRECTORY_SIZE_GUESS = 16*1024; + + private final FileSystem fileSystem; + private final Path path; + private final CompressionKind compressionKind; + private final CompressionCodec codec; + private final int bufferSize; + private final OrcProto.Footer footer; + private final ObjectInspector inspector; + + private static class StripeInformationImpl + implements StripeInformation { + private final OrcProto.StripeInformation stripe; + + StripeInformationImpl(OrcProto.StripeInformation stripe) { + this.stripe = stripe; + } + + @Override + public long getOffset() { + return stripe.getOffset(); + } + + @Override + public long getDataLength() { + return stripe.getDataLength(); + } + + @Override + public long getFooterLength() { + return stripe.getFooterLength(); + } + + @Override + public long getIndexLength() { + return stripe.getIndexLength(); + } + + @Override + public long getNumberOfRows() { + return stripe.getNumberOfRows(); + } + + @Override + public String toString() { + return "offset: " + getOffset() + " data: " + getDataLength() + + " rows: " + getNumberOfRows() + " tail: " + getFooterLength() + + " index: " + getIndexLength(); + } + } + + @Override + public long getNumberOfRows() { + return footer.getNumberOfRows(); + } + + @Override + public Iterable getMetadataKeys() { + List result = new ArrayList(); + for(OrcProto.UserMetadataItem item: footer.getMetadataList()) { + result.add(item.getName()); + } + return result; + } + + @Override + public ByteBuffer getMetadataValue(String key) { + for(OrcProto.UserMetadataItem item: footer.getMetadataList()) { + if (item.hasName() && item.getName().equals(key)) { + return item.getValue().asReadOnlyByteBuffer(); + } + } + throw new IllegalArgumentException("Can't find user metadata " + key); + } + + @Override + public CompressionKind getCompression() { + return compressionKind; + } + + @Override + public int getCompressionSize() { + return bufferSize; + } + + @Override + public Iterable getStripes() { + return new Iterable(){ + + @Override + public Iterator iterator() { + return new Iterator(){ + final Iterator inner = + footer.getStripesList().iterator(); + + @Override + public boolean hasNext() { + return inner.hasNext(); + } + + @Override + public org.apache.hadoop.hive.ql.io.orc.StripeInformation next() { + return new StripeInformationImpl(inner.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove unsupported"); + } + }; + } + }; + } + + @Override + public ObjectInspector getObjectInspector() { + return inspector; + } + + @Override + public long getContentLength() { + return footer.getContentLength(); + } + + @Override + public List getTypes() { + return footer.getTypesList(); + } + + @Override + public int getRowIndexStride() { + return footer.getRowIndexStride(); + } + + @Override + public ColumnStatistics[] getStatistics() { + ColumnStatistics[] result = new ColumnStatistics[footer.getTypesCount()]; + for(int i=0; i < result.length; ++i) { + result[i] = ColumnStatisticsImpl.deserialize(footer.getStatistics(i)); + } + return result; + } + + ReaderImpl(FileSystem fs, Path path) throws IOException { + this.fileSystem = fs; + this.path = path; + FSDataInputStream file = fs.open(path); + long size = fs.getFileStatus(path).getLen(); + int readSize = (int) Math.min(size, DIRECTORY_SIZE_GUESS); + file.seek(size - readSize); + ByteBuffer buffer = ByteBuffer.allocate(readSize); + file.readFully(buffer.array(), buffer.arrayOffset() + buffer.position(), + buffer.remaining()); + int psLen = buffer.get(readSize - 1); + int psOffset = readSize - 1 - psLen; + CodedInputStream in = CodedInputStream.newInstance(buffer.array(), + buffer.arrayOffset() + psOffset, psLen); + OrcProto.PostScript ps = OrcProto.PostScript.parseFrom(in); + int footerSize = (int) ps.getFooterLength(); + bufferSize = (int) ps.getCompressionBlockSize(); + switch (ps.getCompression()) { + case NONE: + compressionKind = CompressionKind.NONE; + break; + case ZLIB: + compressionKind = CompressionKind.ZLIB; + break; + case SNAPPY: + compressionKind = CompressionKind.SNAPPY; + break; + case LZO: + compressionKind = CompressionKind.LZO; + break; + default: + throw new IllegalArgumentException("Unknown compression"); + } + codec = WriterImpl.createCodec(compressionKind); + int extra = Math.max(0, psLen + 1 + footerSize - readSize); + if (extra > 0) { + file.seek(size - readSize - extra); + ByteBuffer extraBuf = ByteBuffer.allocate(extra + readSize); + file.readFully(extraBuf.array(), + extraBuf.arrayOffset() + extraBuf.position(), extra); + extraBuf.position(extra); + extraBuf.put(buffer); + buffer = extraBuf; + buffer.position(0); + buffer.limit(footerSize); + } else { + buffer.position(psOffset - footerSize); + buffer.limit(psOffset); + } + InputStream instream = InStream.create("footer", buffer, codec, bufferSize); + footer = OrcProto.Footer.parseFrom(instream); + inspector = OrcStruct.createObjectInspector(0, footer.getTypesList()); + file.close(); + } + + @Override + public RecordReader rows(boolean include[]) throws IOException { + return rows(0, Long.MAX_VALUE, include); + } + + @Override + public RecordReader rows(long offset, long length, boolean[] include + ) throws IOException { + return new RecordReaderImpl(this.getStripes(), fileSystem, path, offset, + length, footer.getTypesList(), codec, bufferSize, + include); + } + +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java (working copy) @@ -0,0 +1,517 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; + +class ColumnStatisticsImpl implements ColumnStatistics { + + private final static class BooleanStatisticsImpl extends ColumnStatisticsImpl + implements BooleanColumnStatistics { + private long trueCount = 0; + + BooleanStatisticsImpl(OrcProto.ColumnStatistics stats) { + super(stats); + OrcProto.BucketStatistics bkt = stats.getBucketStatistics(); + trueCount = bkt.getCount(1); + } + + BooleanStatisticsImpl() { + } + + @Override + void reset() { + super.reset(); + trueCount = 0; + } + + @Override + void updateBoolean(boolean value) { + if (value) { + trueCount += 1; + } + } + + @Override + void merge(ColumnStatisticsImpl other) { + super.merge(other); + BooleanStatisticsImpl bkt = (BooleanStatisticsImpl) other; + trueCount += bkt.trueCount; + } + + @Override + OrcProto.ColumnStatistics.Builder serialize() { + OrcProto.ColumnStatistics.Builder builder = super.serialize(); + OrcProto.BucketStatistics.Builder bucket = + OrcProto.BucketStatistics.newBuilder(); + bucket.addCount(count - trueCount); + bucket.addCount(trueCount); + builder.setBucketStatistics(bucket); + return builder; + } + + @Override + public long getFalseCount() { + return count - trueCount; + } + + @Override + public long getTrueCount() { + return trueCount; + } + + @Override + public String toString() { + return super.toString() + " true: " + trueCount; + } + } + + private final static class IntegerStatisticsImpl extends ColumnStatisticsImpl + implements IntegerColumnStatistics { + + long minimum = Long.MAX_VALUE; + long maximum = Long.MIN_VALUE; + long sum = 0; + boolean hasMinimum = false; + boolean overflow = false; + + IntegerStatisticsImpl() { + } + + IntegerStatisticsImpl(OrcProto.ColumnStatistics stats) { + super(stats); + OrcProto.IntegerStatistics intStat = stats.getIntStatistics(); + if (intStat.hasMinimum()) { + hasMinimum = true; + minimum = intStat.getMinimum(); + } + if (intStat.hasMaximum()) { + maximum = intStat.getMaximum(); + } + if (intStat.hasSum()) { + sum = intStat.getSum(); + } else { + overflow = true; + } + } + + @Override + void reset() { + super.reset(); + hasMinimum = false; + minimum = Long.MAX_VALUE; + maximum = Long.MIN_VALUE; + sum = 0; + overflow = false; + } + + @Override + void updateInteger(long value) { + if (!hasMinimum) { + hasMinimum = true; + minimum = value; + maximum = value; + } else if (value < minimum) { + minimum = value; + } else if (value > maximum) { + maximum = value; + } + if (!overflow) { + boolean wasPositive = sum >= 0; + sum += value; + if ((value >= 0) == wasPositive) { + overflow = (sum >= 0) != wasPositive; + } + } + } + + @Override + void merge(ColumnStatisticsImpl other) { + IntegerStatisticsImpl otherInt = (IntegerStatisticsImpl) other; + if (!hasMinimum) { + hasMinimum = otherInt.hasMinimum; + minimum = otherInt.minimum; + maximum = otherInt.maximum; + } else if (otherInt.hasMinimum) { + if (otherInt.minimum < minimum) { + minimum = otherInt.minimum; + } + if (otherInt.maximum > maximum) { + maximum = otherInt.maximum; + } + } + super.merge(other); + overflow |= otherInt.overflow; + if (!overflow) { + boolean wasPositive = sum >= 0; + sum += otherInt.sum; + if ((otherInt.sum >= 0) == wasPositive) { + overflow = (sum >= 0) != wasPositive; + } + } + } + + @Override + OrcProto.ColumnStatistics.Builder serialize() { + OrcProto.ColumnStatistics.Builder builder = super.serialize(); + OrcProto.IntegerStatistics.Builder intb = + OrcProto.IntegerStatistics.newBuilder(); + if (hasMinimum) { + intb.setMinimum(minimum); + intb.setMaximum(maximum); + } + if (!overflow) { + intb.setSum(sum); + } + builder.setIntStatistics(intb); + return builder; + } + + @Override + public long getMinimum() { + return minimum; + } + + @Override + public long getMaximum() { + return maximum; + } + + @Override + public boolean isSumDefined() { + return !overflow; + } + + @Override + public long getSum() { + return sum; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(super.toString()); + if (hasMinimum) { + buf.append(" min: "); + buf.append(minimum); + buf.append(" max: "); + buf.append(maximum); + } + if (!overflow) { + buf.append(" sum: "); + buf.append(sum); + } + return buf.toString(); + } + } + + private final static class DoubleStatisticsImpl extends ColumnStatisticsImpl + implements DoubleColumnStatistics { + boolean hasMinimum = false; + double minimum = Double.MAX_VALUE; + double maximum = Double.MIN_VALUE; + double sum = 0; + + DoubleStatisticsImpl() { + } + + DoubleStatisticsImpl(OrcProto.ColumnStatistics stats) { + super(stats); + OrcProto.DoubleStatistics dbl = stats.getDoubleStatistics(); + if (dbl.hasMinimum()) { + hasMinimum = true; + minimum = dbl.getMinimum(); + } + if (dbl.hasMaximum()) { + maximum = dbl.getMaximum(); + } + if (dbl.hasSum()) { + sum = dbl.getSum(); + } + } + + @Override + void reset() { + super.reset(); + hasMinimum = false; + minimum = Double.MAX_VALUE; + maximum = Double.MIN_VALUE; + sum = 0; + } + + @Override + void updateDouble(double value) { + if (!hasMinimum) { + hasMinimum = true; + minimum = value; + maximum = value; + } else if (value < minimum) { + minimum = value; + } else if (value > maximum) { + maximum = value; + } + sum += value; + } + + @Override + void merge(ColumnStatisticsImpl other) { + super.merge(other); + DoubleStatisticsImpl dbl = (DoubleStatisticsImpl) other; + if (!hasMinimum) { + hasMinimum = dbl.hasMinimum; + minimum = dbl.minimum; + maximum = dbl.maximum; + } else if (dbl.hasMinimum) { + if (dbl.minimum < minimum) { + minimum = dbl.minimum; + } + if (dbl.maximum > maximum) { + maximum = dbl.maximum; + } + } + sum += dbl.sum; + } + + @Override + OrcProto.ColumnStatistics.Builder serialize() { + OrcProto.ColumnStatistics.Builder builder = super.serialize(); + OrcProto.DoubleStatistics.Builder dbl = + OrcProto.DoubleStatistics.newBuilder(); + if (hasMinimum) { + dbl.setMinimum(minimum); + dbl.setMaximum(maximum); + } + dbl.setSum(sum); + builder.setDoubleStatistics(dbl); + return builder; + } + + @Override + public double getMinimum() { + return minimum; + } + + @Override + public double getMaximum() { + return maximum; + } + + @Override + public double getSum() { + return sum; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(super.toString()); + if (hasMinimum) { + buf.append(" min: "); + buf.append(minimum); + buf.append(" max: "); + buf.append(maximum); + } + buf.append(" sum: "); + buf.append(sum); + return buf.toString(); + } + } + + private final static class StringStatisticsImpl extends ColumnStatisticsImpl + implements StringColumnStatistics { + String minimum = null; + String maximum = null; + + StringStatisticsImpl() { + } + + StringStatisticsImpl(OrcProto.ColumnStatistics stats) { + super(stats); + OrcProto.StringStatistics str = stats.getStringStatistics(); + if (str.hasMaximum()) { + maximum = str.getMaximum(); + } + if (str.hasMinimum()) { + minimum = str.getMinimum(); + } + } + + @Override + void reset() { + super.reset(); + minimum = null; + maximum = null; + } + + @Override + void updateString(String value) { + if (minimum == null) { + minimum = value; + maximum = value; + } else if (minimum.compareTo(value) > 0) { + minimum = value; + } else if (maximum.compareTo(value) < 0) { + maximum = value; + } + } + + @Override + void merge(ColumnStatisticsImpl other) { + super.merge(other); + StringStatisticsImpl str = (StringStatisticsImpl) other; + if (minimum == null) { + minimum = str.minimum; + maximum = str.maximum; + } else if (str.minimum != null) { + if (minimum.compareTo(str.minimum) > 0) { + minimum = str.minimum; + } else if (maximum.compareTo(str.maximum) < 0) { + maximum = str.maximum; + } + } + } + + @Override + OrcProto.ColumnStatistics.Builder serialize() { + OrcProto.ColumnStatistics.Builder result = super.serialize(); + OrcProto.StringStatistics.Builder str = + OrcProto.StringStatistics.newBuilder(); + if (count != 0) { + str.setMinimum(minimum); + str.setMaximum(maximum); + } + result.setStringStatistics(str); + return result; + } + + @Override + public String getMinimum() { + return minimum; + } + + @Override + public String getMaximum() { + return maximum; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(super.toString()); + if (count != 0) { + buf.append(" min: "); + buf.append(minimum); + buf.append(" max: "); + buf.append(maximum); + } + return buf.toString(); + } + } + + protected long count = 0; + + ColumnStatisticsImpl(OrcProto.ColumnStatistics stats) { + if (stats.hasNumberOfValues()) { + count = stats.getNumberOfValues(); + } + } + + ColumnStatisticsImpl() { + } + + void increment() { + count += 1; + } + + void updateBoolean(boolean value) { + throw new UnsupportedOperationException("Can't update boolean"); + } + + void updateInteger(long value) { + throw new UnsupportedOperationException("Can't update integer"); + } + + void updateDouble(double value) { + throw new UnsupportedOperationException("Can't update double"); + } + + void updateString(String value) { + throw new UnsupportedOperationException("Can't update string"); + } + + void merge(ColumnStatisticsImpl stats) { + count += stats.count; + } + + void reset() { + count = 0; + } + + @Override + public long getNumberOfValues() { + return count; + } + + @Override + public String toString() { + return "count: " + count; + } + + OrcProto.ColumnStatistics.Builder serialize() { + OrcProto.ColumnStatistics.Builder builder = + OrcProto.ColumnStatistics.newBuilder(); + builder.setNumberOfValues(count); + return builder; + } + + static ColumnStatisticsImpl create(ObjectInspector inspector) { + switch (inspector.getCategory()) { + case PRIMITIVE: + switch (((PrimitiveObjectInspector) inspector).getPrimitiveCategory()) { + case BOOLEAN: + return new BooleanStatisticsImpl(); + case BYTE: + case SHORT: + case INT: + case LONG: + return new IntegerStatisticsImpl(); + case FLOAT: + case DOUBLE: + return new DoubleStatisticsImpl(); + case STRING: + return new StringStatisticsImpl(); + default: + return new ColumnStatisticsImpl(); + } + default: + return new ColumnStatisticsImpl(); + } + } + + static ColumnStatisticsImpl deserialize(OrcProto.ColumnStatistics stats) { + if (stats.hasBucketStatistics()) { + return new BooleanStatisticsImpl(stats); + } else if (stats.hasIntStatistics()) { + return new IntegerStatisticsImpl(stats); + } else if (stats.hasDoubleStatistics()) { + return new DoubleStatisticsImpl(stats); + } else if (stats.hasStringStatistics()) { + return new StringStatisticsImpl(stats); + } else { + return new ColumnStatisticsImpl(stats); + } + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/PositionRecorder.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/PositionRecorder.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/PositionRecorder.java (working copy) @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +interface PositionRecorder { + void addPosition(long offset); +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (working copy) @@ -0,0 +1,1249 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.io.BytesWritable; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +class WriterImpl implements Writer { + + private static final int HDFS_BUFFER_SIZE = 256 * 1024; + private static final int MIN_ROW_INDEX_STRIDE = 1000; + + private final FileSystem fs; + private final Path path; + private final long stripeSize; + private final int rowIndexStride; + private final CompressionKind compress; + private final CompressionCodec codec; + private final int bufferSize; + // how many compressed bytes in the current stripe so far + private long bytesInStripe = 0; + // the streams that make up the current stripe + private final Map streams = + new TreeMap(); + + private FSDataOutputStream rawWriter = null; + // the compressed metadata information outStream + private OutStream writer = null; + // a protobuf outStream around streamFactory + private CodedOutputStream protobufWriter = null; + private long headerLength; + private int columnCount; + private long rowCount = 0; + private long rowsInStripe = 0; + private int rowsInIndex = 0; + private final List stripes = + new ArrayList(); + private final Map userMetadata = + new TreeMap(); + private final StreamFactory StreamFactory = new StreamFactory(); + private final TreeWriter treeWriter; + private final OrcProto.RowIndex.Builder rowIndex = + OrcProto.RowIndex.newBuilder(); + private final boolean buildIndex; + + WriterImpl(FileSystem fs, + Path path, + ObjectInspector inspector, + long stripeSize, + CompressionKind compress, + int bufferSize, + int rowIndexStride) throws IOException { + this.fs = fs; + this.path = path; + this.stripeSize = stripeSize; + this.compress = compress; + this.bufferSize = bufferSize; + this.rowIndexStride = rowIndexStride; + buildIndex = rowIndexStride > 0; + codec = createCodec(compress); + treeWriter = createTreeWriter(inspector, StreamFactory, false); + if (buildIndex && rowIndexStride < MIN_ROW_INDEX_STRIDE) { + throw new IllegalArgumentException("Row stride must be at least " + + MIN_ROW_INDEX_STRIDE); + } + } + + static CompressionCodec createCodec(CompressionKind kind) { + switch (kind) { + case NONE: + return null; + case ZLIB: + return new ZlibCodec(); + case SNAPPY: + return new SnappyCodec(); + case LZO: + try { + Class lzo = + (Class) + Class.forName("org.apache.hadoop.hive.ql.io.orc.LzoCodec"); + return lzo.newInstance(); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("LZO is not available.", e); + } catch (InstantiationException e) { + throw new IllegalArgumentException("Problem initializing LZO", e); + } catch (IllegalAccessException e) { + throw new IllegalArgumentException("Insufficient access to LZO", e); + } + default: + throw new IllegalArgumentException("Unknown compression codec: " + + kind); + } + } + + private class BufferedStream implements OutStream.OutputReceiver { + final OutStream outStream; + final List output = new ArrayList(); + + BufferedStream(String name, int bufferSize, + CompressionCodec codec) throws IOException { + outStream = new OutStream(name, bufferSize, codec, this); + } + + @Override + public void output(ByteBuffer buffer) throws IOException { + output.add(buffer); + bytesInStripe += buffer.remaining(); + } + + public void flush() throws IOException { + outStream.flush(); + } + + public void clear() throws IOException { + outStream.clear(); + output.clear(); + } + + @Override + public long getPosition() { + long result = 0; + for(ByteBuffer buffer: output) { + result += buffer.remaining(); + } + return result; + } + + void spillTo(OutputStream out) throws IOException { + for(ByteBuffer buffer: output) { + out.write(buffer.array(), buffer.arrayOffset() + buffer.position(), + buffer.remaining()); + } + } + } + + private class DirectStream implements OutStream.OutputReceiver { + final FSDataOutputStream output; + + DirectStream(FSDataOutputStream output) { + this.output = output; + } + + @Override + public void output(ByteBuffer buffer) throws IOException { + output.write(buffer.array(), buffer.arrayOffset() + buffer.position(), + buffer.remaining()); + } + + @Override + public long getPosition() throws IOException { + return output.getPos(); + } + } + + private static class RowIndexPositionRecorder implements PositionRecorder { + private final OrcProto.RowIndexEntry.Builder builder; + + RowIndexPositionRecorder(OrcProto.RowIndexEntry.Builder builder) { + this.builder = builder; + } + + @Override + public void addPosition(long position) { + builder.addPositions(position); + } + } + + private class StreamFactory { + /** + * Create a stream to store part of a column + * @param column the column id for the stream + * @param kind the kind of stream + * @return The output outStream that the section needs to be written to. + * @throws IOException + */ + public PositionedOutputStream createStream(int column, + OrcProto.Stream.Kind kind + ) throws IOException { + StreamName name = new StreamName(column, kind); + BufferedStream result = streams.get(name); + if (result == null) { + result = new BufferedStream(name.toString(), bufferSize, codec); + streams.put(name, result); + } + return result.outStream; + } + + /** + * Get the next column id. + * @return a number from 0 to the number of columns - 1 + */ + public int getNextColumnId() { + return columnCount++; + } + + /** + * Get the stride rate of the row index. + */ + public int getRowIndexStride() { + return rowIndexStride; + } + + /** + * Should be building the row index + * @return true if we are building the index + */ + public boolean buildIndex() { + return buildIndex; + } + } + + private static abstract class TreeWriter { + protected final int id; + protected final ObjectInspector inspector; + protected final StreamFactory streamFactory; + private final BitFieldWriter isPresent; + protected final ColumnStatisticsImpl stripeStatistics; + protected final ColumnStatisticsImpl fileStatistics; + protected TreeWriter[] childrenWriters; + protected final RowIndexPositionRecorder rowIndexPosition; + protected final OrcProto.RowIndex.Builder rowIndex; + protected final OrcProto.RowIndexEntry.Builder rowIndexEntry; + private final PositionedOutputStream rowIndexStream; + + TreeWriter(int columnId, ObjectInspector inspector, + StreamFactory streamFactory, + boolean nullable) throws IOException { + this.id = columnId; + this.inspector = inspector; + this.streamFactory = streamFactory; + if (nullable) { + isPresent = new BitFieldWriter(streamFactory.createStream(id, + OrcProto.Stream.Kind.PRESENT), 1); + } else { + isPresent = null; + } + stripeStatistics = ColumnStatisticsImpl.create(inspector); + fileStatistics = ColumnStatisticsImpl.create(inspector); + childrenWriters = new TreeWriter[0]; + rowIndex = OrcProto.RowIndex.newBuilder(); + rowIndexEntry = OrcProto.RowIndexEntry.newBuilder(); + rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry); + if (streamFactory.buildIndex()) { + rowIndexStream = streamFactory.createStream(id, + OrcProto.Stream.Kind.ROW_INDEX); + } else { + rowIndexStream = null; + } + } + + void write(Object obj) throws IOException { + if (obj != null) { + stripeStatistics.increment(); + } + if (isPresent != null) { + isPresent.append(obj == null ? 0: 1); + } + } + + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + if (isPresent != null) { + isPresent.flush(); + } + builder.addColumns(getEncoding()); + if (rowIndexStream != null) { + rowIndex.build().writeTo(rowIndexStream); + rowIndexStream.flush(); + } + rowIndex.clear(); + rowIndexEntry.clear(); + } + + TreeWriter[] getChildrenWriters() { + return childrenWriters; + } + + OrcProto.ColumnEncoding getEncoding() { + return OrcProto.ColumnEncoding.DIRECT; + } + + void createRowIndexEntry() throws IOException { + fileStatistics.merge(stripeStatistics); + rowIndexEntry.setStatistics(stripeStatistics.serialize()); + stripeStatistics.reset(); + rowIndex.addEntry(rowIndexEntry); + rowIndexEntry.clear(); + recordPosition(rowIndexPosition); + for(TreeWriter child: childrenWriters) { + child.createRowIndexEntry(); + } + } + + void recordPosition(PositionRecorder recorder) throws IOException { + if (isPresent != null) { + isPresent.getPosition(recorder); + } + } + } + + private static class BooleanTreeWriter extends TreeWriter { + private final BitFieldWriter writer; + + BooleanTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + PositionedOutputStream out = writer.createStream(id, + OrcProto.Stream.Kind.DATA); + this.writer = new BitFieldWriter(out, 1); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + boolean val = ((BooleanObjectInspector) inspector).get(obj); + stripeStatistics.updateBoolean(val); + writer.append(val ? 1 : 0); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + writer.flush(); + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + writer.getPosition(recorder); + } + } + + private static class ByteTreeWriter extends TreeWriter { + private final RunLengthByteWriter writer; + + ByteTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + this.writer = new RunLengthByteWriter(writer.createStream(id, + OrcProto.Stream.Kind.DATA)); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + byte val = ((ByteObjectInspector) inspector).get(obj); + stripeStatistics.updateInteger(val); + writer.write(val); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + writer.flush(); + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + writer.getPosition(recorder); + } + } + + private static class IntegerTreeWriter extends TreeWriter { + private final RunLengthIntegerWriter writer; + private final ShortObjectInspector shortInspector; + private final IntObjectInspector intInspector; + private final LongObjectInspector longInspector; + + IntegerTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + PositionedOutputStream out = writer.createStream(id, + OrcProto.Stream.Kind.DATA); + this.writer = new RunLengthIntegerWriter(out, true); + if (inspector instanceof IntObjectInspector) { + intInspector = (IntObjectInspector) inspector; + shortInspector = null; + longInspector = null; + } else { + intInspector = null; + if (inspector instanceof LongObjectInspector) { + longInspector = (LongObjectInspector) inspector; + shortInspector = null; + } else { + shortInspector = (ShortObjectInspector) inspector; + longInspector = null; + } + } + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + long val; + if (intInspector != null) { + val = intInspector.get(obj); + } else if (longInspector != null) { + val = longInspector.get(obj); + } else { + val = shortInspector.get(obj); + } + stripeStatistics.updateInteger(val); + writer.write(val); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + writer.flush(); + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + writer.getPosition(recorder); + } + } + + private static class FloatTreeWriter extends TreeWriter { + private final PositionedOutputStream stream; + + FloatTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + this.stream = writer.createStream(id, + OrcProto.Stream.Kind.DATA); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + float val = ((FloatObjectInspector) inspector).get(obj); + stripeStatistics.updateDouble(val); + SerializationUtils.writeFloat(stream, val); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + stream.flush(); + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + stream.getPosition(recorder); + } + } + + private static class DoubleTreeWriter extends TreeWriter { + private final PositionedOutputStream stream; + + DoubleTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + this.stream = writer.createStream(id, + OrcProto.Stream.Kind.DATA); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + double val = ((DoubleObjectInspector) inspector).get(obj); + stripeStatistics.updateDouble(val); + SerializationUtils.writeDouble(stream, val); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + stream.flush(); + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + stream.getPosition(recorder); + } + } + + private static class StringTreeWriter extends TreeWriter { + private final PositionedOutputStream stringOutput; + private final RunLengthIntegerWriter lengthOutput; + private final RunLengthIntegerWriter rowOutput; + private final RunLengthIntegerWriter countOutput; + private final StringRedBlackTree dictionary = new StringRedBlackTree(); + private final DynamicIntArray rows = new DynamicIntArray(); + private final List savedRowIndex = + new ArrayList(); + private final int rowIndexStride; + private final boolean buildIndex; + + StringTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + stringOutput = writer.createStream(id, + OrcProto.Stream.Kind.DICTIONARY_DATA); + lengthOutput = new RunLengthIntegerWriter(writer.createStream(id, + OrcProto.Stream.Kind.LENGTH), false); + rowOutput = new RunLengthIntegerWriter(writer.createStream(id, + OrcProto.Stream.Kind.DATA), false); + if (writer.buildIndex()) { + countOutput = new RunLengthIntegerWriter(writer.createStream(id, + OrcProto.Stream.Kind.DICTIONARY_COUNT), false); + } else { + countOutput = null; + } + recordPosition(rowIndexPosition); + rowIndexStride = writer.getRowIndexStride(); + buildIndex = writer.buildIndex(); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + String val = ((StringObjectInspector) inspector) + .getPrimitiveJavaObject(obj); + rows.add(dictionary.add(val)); + stripeStatistics.updateString(val); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + final int[] dumpOrder = new int[dictionary.size()]; + dictionary.visit(new StringRedBlackTree.Visitor() { + int currentId = 0; + @Override + public void visit(StringRedBlackTree.VisitorContext context + ) throws IOException { + context.writeBytes(stringOutput); + lengthOutput.write(context.getLength()); + dumpOrder[context.getOriginalPosition()] = currentId++; + if (countOutput != null) { + countOutput.write(context.getCount()); + } + } + }); + int length = rows.size(); + int rowIndexEntry = 0; + for(int i=0; i < length; ++i) { + rowOutput.write(dumpOrder[rows.get(i)]); + // now that we are writing out the row values, we can finalize the + // row index + if (buildIndex && i % rowIndexStride == 0) { + OrcProto.RowIndexEntry.Builder base = + savedRowIndex.get(rowIndexEntry++).toBuilder(); + rowOutput.getPosition(new RowIndexPositionRecorder(base)); + rowIndex.addEntry(base.build()); + } + } + // we need to build the rowindex before calling super, since it + // writes it out. + super.writeStripe(builder); + stringOutput.flush(); + lengthOutput.flush(); + rowOutput.flush(); + if (countOutput != null) { + countOutput.flush(); + } + dictionary.clear(); + rows.clear(); + savedRowIndex.clear(); + recordPosition(rowIndexPosition); + } + + @Override + OrcProto.ColumnEncoding getEncoding() { + return OrcProto.ColumnEncoding.DICTIONARY; + } + + void createRowIndexEntry() throws IOException { + fileStatistics.merge(stripeStatistics); + rowIndexEntry.setStatistics(stripeStatistics.serialize()); + stripeStatistics.reset(); + savedRowIndex.add(rowIndexEntry.build()); + rowIndexEntry.clear(); + recordPosition(rowIndexPosition); + } + } + + private static class BinaryTreeWriter extends TreeWriter { + private final PositionedOutputStream stream; + private final RunLengthIntegerWriter length; + + BinaryTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + this.stream = writer.createStream(id, + OrcProto.Stream.Kind.DATA); + this.length = new RunLengthIntegerWriter(writer.createStream(id, + OrcProto.Stream.Kind.LENGTH), false); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + BytesWritable val = + ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj); + stream.write(val.getBytes(), 0, val.getLength()); + length.write(val.getLength()); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + stream.flush(); + length.flush(); + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + stream.getPosition(recorder); + length.getPosition(recorder); + } + } + + static final int MILLIS_PER_SECOND = 1000; + static final long BASE_TIMESTAMP = + Timestamp.valueOf("2015-01-01 00:00:00").getTime() / MILLIS_PER_SECOND; + + private static class TimestampTreeWriter extends TreeWriter { + private final RunLengthIntegerWriter seconds; + private final RunLengthIntegerWriter nanos; + + TimestampTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + this.seconds = new RunLengthIntegerWriter(writer.createStream(id, + OrcProto.Stream.Kind.DATA), true); + this.nanos = new RunLengthIntegerWriter(writer.createStream(id, + OrcProto.Stream.Kind.NANO_DATA), false); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + Timestamp val = + ((TimestampObjectInspector) inspector). + getPrimitiveJavaObject(obj); + seconds.write((val.getTime() / MILLIS_PER_SECOND) - BASE_TIMESTAMP); + nanos.write(formatNanos(val.getNanos())); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + seconds.flush(); + nanos.flush(); + recordPosition(rowIndexPosition); + } + + private static long formatNanos(int nanos) { + if (nanos == 0) { + return 0; + } else if (nanos % 100 != 0) { + return ((long) nanos) << 3; + } else { + nanos /= 100; + int trailingZeros = 1; + while (nanos % 10 == 0 && trailingZeros < 7) { + nanos /= 10; + trailingZeros += 1; + } + return ((long) nanos) << 3 | trailingZeros; + } + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + seconds.getPosition(recorder); + nanos.getPosition(recorder); + } + } + + private static class StructTreeWriter extends TreeWriter { + private final List fields; + StructTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + StructObjectInspector structObjectInspector = + (StructObjectInspector) inspector; + fields = structObjectInspector.getAllStructFieldRefs(); + childrenWriters = new TreeWriter[fields.size()]; + for(int i=0; i < childrenWriters.length; ++i) { + childrenWriters[i] = createTreeWriter( + fields.get(i).getFieldObjectInspector(), writer, true); + } + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + StructObjectInspector insp = (StructObjectInspector) inspector; + for(int i = 0; i < fields.size(); ++i) { + StructField field = fields.get(i); + TreeWriter writer = childrenWriters[i]; + writer.write(insp.getStructFieldData(obj, field)); + } + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + for(TreeWriter child: childrenWriters) { + child.writeStripe(builder); + } + recordPosition(rowIndexPosition); + } + } + + private static class ListTreeWriter extends TreeWriter { + private final RunLengthIntegerWriter lengths; + + ListTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + ListObjectInspector listObjectInspector = (ListObjectInspector) inspector; + childrenWriters = new TreeWriter[1]; + childrenWriters[0] = + createTreeWriter(listObjectInspector.getListElementObjectInspector(), + writer, true); + lengths = + new RunLengthIntegerWriter(writer.createStream(columnId, + OrcProto.Stream.Kind.LENGTH), false); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + ListObjectInspector insp = (ListObjectInspector) inspector; + int len = insp.getListLength(obj); + lengths.write(len); + for(int i=0; i < len; ++i) { + childrenWriters[0].write(insp.getListElement(obj, i)); + } + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + lengths.flush(); + for(TreeWriter child: childrenWriters) { + child.writeStripe(builder); + } + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + lengths.getPosition(recorder); + } + } + + private static class MapTreeWriter extends TreeWriter { + private final RunLengthIntegerWriter lengths; + + MapTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + MapObjectInspector insp = (MapObjectInspector) inspector; + childrenWriters = new TreeWriter[2]; + childrenWriters[0] = + createTreeWriter(insp.getMapKeyObjectInspector(), writer, true); + childrenWriters[1] = + createTreeWriter(insp.getMapValueObjectInspector(), writer, true); + lengths = + new RunLengthIntegerWriter(writer.createStream(columnId, + OrcProto.Stream.Kind.LENGTH), false); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + MapObjectInspector insp = (MapObjectInspector) inspector; + int len = insp.getMapSize(obj); + lengths.write(len); + // this sucks, but it will have to do until we can get a better + // accessor in the MapObjectInspector. + Map valueMap = insp.getMap(obj); + for(Map.Entry entry: valueMap.entrySet()) { + childrenWriters[0].write(entry.getKey()); + childrenWriters[1].write(entry.getValue()); + } + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + lengths.flush(); + for(TreeWriter child: childrenWriters) { + child.writeStripe(builder); + } + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + lengths.getPosition(recorder); + } + } + + private static class UnionTreeWriter extends TreeWriter { + private final RunLengthByteWriter tags; + + UnionTreeWriter(int columnId, + ObjectInspector inspector, + StreamFactory writer, + boolean nullable) throws IOException { + super(columnId, inspector, writer, nullable); + UnionObjectInspector insp = (UnionObjectInspector) inspector; + List choices = insp.getObjectInspectors(); + childrenWriters = new TreeWriter[choices.size()]; + for(int i=0; i < childrenWriters.length; ++i) { + childrenWriters[i] = createTreeWriter(choices.get(i), writer, true); + } + tags = + new RunLengthByteWriter(writer.createStream(columnId, + OrcProto.Stream.Kind.DATA)); + recordPosition(rowIndexPosition); + } + + @Override + void write(Object obj) throws IOException { + super.write(obj); + if (obj != null) { + UnionObjectInspector insp = (UnionObjectInspector) inspector; + byte tag = insp.getTag(obj); + tags.write(tag); + childrenWriters[tag].write(insp.getField(obj)); + } + } + + @Override + void writeStripe(OrcProto.StripeFooter.Builder builder) throws IOException { + super.writeStripe(builder); + tags.flush(); + for(TreeWriter child: childrenWriters) { + child.writeStripe(builder); + } + recordPosition(rowIndexPosition); + } + + @Override + void recordPosition(PositionRecorder recorder) throws IOException { + super.recordPosition(recorder); + tags.getPosition(recorder); + } + } + + private static TreeWriter createTreeWriter(ObjectInspector inspector, + StreamFactory streamFactory, + boolean nullable + ) throws IOException { + switch (inspector.getCategory()) { + case PRIMITIVE: + switch (((PrimitiveObjectInspector) inspector).getPrimitiveCategory()) { + case BOOLEAN: + return new BooleanTreeWriter(streamFactory.getNextColumnId(), + inspector, streamFactory, nullable); + case BYTE: + return new ByteTreeWriter(streamFactory.getNextColumnId(), + inspector, streamFactory, nullable); + case SHORT: + case INT: + case LONG: + return new IntegerTreeWriter(streamFactory.getNextColumnId(), + inspector, streamFactory, nullable); + case FLOAT: + return new FloatTreeWriter(streamFactory.getNextColumnId(), + inspector, streamFactory, nullable); + case DOUBLE: + return new DoubleTreeWriter(streamFactory.getNextColumnId(), + inspector, streamFactory, nullable); + case STRING: + return new StringTreeWriter(streamFactory.getNextColumnId(), + inspector, streamFactory, nullable); + case BINARY: + return new BinaryTreeWriter(streamFactory.getNextColumnId(), + inspector, streamFactory, nullable); + case TIMESTAMP: + return new TimestampTreeWriter(streamFactory.getNextColumnId(), + inspector, streamFactory, nullable); + default: + throw new IllegalArgumentException("Bad primitive category " + + ((PrimitiveObjectInspector) inspector).getPrimitiveCategory()); + } + case STRUCT: + return new StructTreeWriter(streamFactory.getNextColumnId(), inspector, + streamFactory, nullable); + case MAP: + return new MapTreeWriter(streamFactory.getNextColumnId(), inspector, + streamFactory, nullable); + case LIST: + return new ListTreeWriter(streamFactory.getNextColumnId(), inspector, + streamFactory, nullable); + case UNION: + return new UnionTreeWriter(streamFactory.getNextColumnId(), inspector, + streamFactory, nullable); + default: + throw new IllegalArgumentException("Bad category: " + + inspector.getCategory()); + } + } + + private static void writeTypes(OrcProto.Footer.Builder builder, + TreeWriter treeWriter) { + OrcProto.Type.Builder type = OrcProto.Type.newBuilder(); + switch (treeWriter.inspector.getCategory()) { + case PRIMITIVE: + switch (((PrimitiveObjectInspector) treeWriter.inspector). + getPrimitiveCategory()) { + case BOOLEAN: + type.setKind(OrcProto.Type.Kind.BOOLEAN); + break; + case BYTE: + type.setKind(OrcProto.Type.Kind.BYTE); + break; + case SHORT: + type.setKind(OrcProto.Type.Kind.SHORT); + break; + case INT: + type.setKind(OrcProto.Type.Kind.INT); + break; + case LONG: + type.setKind(OrcProto.Type.Kind.LONG); + break; + case FLOAT: + type.setKind(OrcProto.Type.Kind.FLOAT); + break; + case DOUBLE: + type.setKind(OrcProto.Type.Kind.DOUBLE); + break; + case STRING: + type.setKind(OrcProto.Type.Kind.STRING); + break; + case BINARY: + type.setKind(OrcProto.Type.Kind.BINARY); + break; + case TIMESTAMP: + type.setKind(OrcProto.Type.Kind.TIMESTAMP); + break; + default: + throw new IllegalArgumentException("Unknown primitive category: " + + ((PrimitiveObjectInspector) treeWriter.inspector). + getPrimitiveCategory()); + } + break; + case LIST: + type.setKind(OrcProto.Type.Kind.LIST); + type.addSubtypes(treeWriter.childrenWriters[0].id); + break; + case MAP: + type.setKind(OrcProto.Type.Kind.MAP); + type.addSubtypes(treeWriter.childrenWriters[0].id); + type.addSubtypes(treeWriter.childrenWriters[1].id); + break; + case STRUCT: + type.setKind(OrcProto.Type.Kind.STRUCT); + for(TreeWriter child: treeWriter.childrenWriters) { + type.addSubtypes(child.id); + } + for(StructField field: ((StructTreeWriter) treeWriter).fields) { + type.addFieldNames(field.getFieldName()); + } + break; + case UNION: + type.setKind(OrcProto.Type.Kind.UNION); + for(TreeWriter child: treeWriter.childrenWriters) { + type.addSubtypes(child.id); + } + break; + default: + throw new IllegalArgumentException("Unknown category: " + + treeWriter.inspector.getCategory()); + } + builder.addTypes(type); + for(TreeWriter child: treeWriter.childrenWriters) { + writeTypes(builder, child); + } + } + + private void ensureWriter() throws IOException { + if (rawWriter == null) { + rawWriter = fs.create(path, false, HDFS_BUFFER_SIZE, + fs.getDefaultReplication(), + Math.min(stripeSize * 2L, Integer.MAX_VALUE)); + rawWriter.writeBytes(OrcFile.MAGIC); + headerLength = rawWriter.getPos(); + writer = new OutStream("metadata", bufferSize, codec, + new DirectStream(rawWriter)); + protobufWriter = CodedOutputStream.newInstance(writer); + } + } + + private void createRowIndexEntry() throws IOException { + treeWriter.createRowIndexEntry(); + rowsInStripe += rowsInIndex; + rowsInIndex = 0; + } + + private void flushStripe() throws IOException { + ensureWriter(); + if (buildIndex && rowsInIndex != 0) { + createRowIndexEntry(); + } + if (rowsInStripe != 0) { + OrcProto.StripeFooter.Builder builder = OrcProto.StripeFooter.newBuilder(); + treeWriter.writeStripe(builder); + long start = rawWriter.getPos(); + long section = start; + long indexEnd = start; + for(Map.Entry pair: streams.entrySet()) { + BufferedStream stream = pair.getValue(); + stream.flush(); + stream.spillTo(rawWriter); + stream.clear(); + long end = rawWriter.getPos(); + StreamName name = pair.getKey(); + builder.addStreams(OrcProto.Stream.newBuilder() + .setColumn(name.getColumn()) + .setKind(name.getKind()) + .setLength(end-section)); + section = end; + if (StreamName.Area.INDEX == name.getArea()) { + indexEnd = end; + } + } + builder.build().writeTo(protobufWriter); + protobufWriter.flush(); + writer.flush(); + long end = rawWriter.getPos(); + OrcProto.StripeInformation dirEntry = + OrcProto.StripeInformation.newBuilder() + .setOffset(start) + .setIndexLength(indexEnd - start) + .setDataLength(section - indexEnd) + .setNumberOfRows(rowsInStripe) + .setFooterLength(end - section).build(); + stripes.add(dirEntry); + rowCount += rowsInStripe; + rowsInStripe = 0; + bytesInStripe = 0; + } + } + + private OrcProto.CompressionKind writeCompressionKind(CompressionKind kind) { + switch (kind) { + case NONE: return OrcProto.CompressionKind.NONE; + case ZLIB: return OrcProto.CompressionKind.ZLIB; + case SNAPPY: return OrcProto.CompressionKind.SNAPPY; + case LZO: return OrcProto.CompressionKind.LZO; + default: + throw new IllegalArgumentException("Unknown compression " + kind); + } + } + + private void writeFileStatistics(OrcProto.Footer.Builder builder, + TreeWriter writer) throws IOException { + builder.addStatistics(writer.fileStatistics.serialize()); + for(TreeWriter child: writer.getChildrenWriters()) { + writeFileStatistics(builder, child); + } + } + + private int writeFooter(long bodyLength) throws IOException { + ensureWriter(); + OrcProto.Footer.Builder builder = OrcProto.Footer.newBuilder(); + builder.setContentLength(bodyLength); + builder.setHeaderLength(headerLength); + builder.setNumberOfRows(rowCount); + builder.setRowIndexStride(rowIndexStride); + // serialize the types + writeTypes(builder, treeWriter); + // add the stripe information + for(OrcProto.StripeInformation stripe: stripes) { + builder.addStripes(stripe); + } + // add the column statistics + writeFileStatistics(builder, treeWriter); + // add all of the user metadata + for(Map.Entry entry: userMetadata.entrySet()) { + builder.addMetadata(OrcProto.UserMetadataItem.newBuilder() + .setName(entry.getKey()).setValue(entry.getValue())); + } + long startPosn = rawWriter.getPos(); + builder.build().writeTo(protobufWriter); + protobufWriter.flush(); + writer.flush(); + return (int) (rawWriter.getPos() - startPosn); + } + + private int writePostScript(int footerLength) throws IOException { + OrcProto.PostScript.Builder builder = + OrcProto.PostScript.newBuilder() + .setCompression(writeCompressionKind(compress)) + .setFooterLength(footerLength); + if (compress != CompressionKind.NONE) { + builder.setCompressionBlockSize(bufferSize); + } + OrcProto.PostScript ps = builder.build(); + // need to write this uncompressed + long startPosn = rawWriter.getPos(); + ps.writeTo(rawWriter); + long length = rawWriter.getPos() - startPosn; + if (length > 255) { + throw new IllegalArgumentException("PostScript too large at " + length); + } + return (int) length; + } + + @Override + public void addUserMetadata(String name, ByteBuffer value) { + userMetadata.put(name, ByteString.copyFrom(value)); + } + + @Override + public void addRow(Object row) throws IOException { + treeWriter.write(row); + if (buildIndex) { + rowsInIndex += 1; + if (buildIndex && rowsInIndex >= rowIndexStride) { + createRowIndexEntry(); + } + } else { + rowsInStripe += 1; + } + if (bytesInStripe >= stripeSize) { + flushStripe(); + } + } + + @Override + public void close() throws IOException { + flushStripe(); + int footerLength = writeFooter(rawWriter.getPos()); + rawWriter.writeByte(writePostScript(footerLength)); + rawWriter.close(); + } +} Index: contrib/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicIntArray.java =================================================================== --- contrib/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicIntArray.java (revision 0) +++ contrib/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicIntArray.java (working copy) @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +/** + * simplistic dynamic array that differentiates from ArrayList by + * - copying pointers to chunks of ints instead of copying the ints + * - managing primitive 'int' types, i.e. not requiring box objects + * + * the motivation for this class is memory optimization, i.e. space efficient + * storage of potentially huge arrays without good a-priori size guesses + * + * the API of this class is between a primitive array and a AbstractList. It's + * not a Collection implementation because it handles primitive types, but the + * API could be extended to support iterators and the like. + * + * NOTE: like standard Collection implementations/arrays, this class is not + * synchronized + */ +class DynamicIntArray { + final static int DEFAULT_CHUNKSIZE = 8 * 1024; + final static int INIT_CHUNKS = 128; + + final int chunkSize; /** our allocation sizes */ + int[][] data; /** the real data */ + int length; /** max set element index +1 */ + int initializedChunks = 0; + + public DynamicIntArray () { + this( DEFAULT_CHUNKSIZE); + } + + public DynamicIntArray (int chunkSize) { + this.chunkSize = chunkSize; + + data = new int[INIT_CHUNKS][]; + } + + /** + * Ensure that the given index is valid. + */ + private void grow(int chunkIndex) { + if (chunkIndex >= initializedChunks) { + if (chunkIndex >= data.length) { + int new_size = Math.max(chunkIndex, 2 * data.length); + int[][] newChunk = new int[new_size][]; + System.arraycopy(data, 0, newChunk, 0, data.length); + data = newChunk; + } + for(int i=initializedChunks; i <= chunkIndex; ++i) { + data[i] = new int[chunkSize]; + } + initializedChunks = chunkIndex + 1; + } + } + + public int get (int index) { + if (index >= length) { + throw new IndexOutOfBoundsException("Index " + index + + " is outside of 0.." + + (length - 1)); + } + int i = index / chunkSize; + int j = index % chunkSize; + return data[i][j]; + } + + public void set (int index, int value) { + int i = index / chunkSize; + int j = index % chunkSize; + grow(i); + if (index >= length) { + length = index+1; + } + data[i][j] = value; + } + + public void increment(int index, int value) { + int i = index / chunkSize; + int j = index % chunkSize; + grow(i); + if (index >= length) { + length = index + 1; + } + data[i][j] += value; + } + + public void add(int value) { + int i = length / chunkSize; + int j = length % chunkSize; + grow(i); + data[i][j] = value; + length += 1; + } + + public int size() { + return length; + } + + public void clear() { + length = 0; + for(int i=0; i < data.length; ++i) { + data[i] = null; + } + initializedChunks = 0; + } + + public String toString() { + int i; + StringBuilder sb = new StringBuilder(length*4); + + sb.append('{'); + int l = length-1; + for (i=0; i>>= 7; + } + } + } + + static void writeVslong(OutputStream output, long value) throws IOException { + writeVulong(output, (value << 1) ^ (value >> 63)); + } + + + static long readVulong(InputStream in) throws IOException { + long result = 0; + long b; + int offset = 0; + do { + b = in.read(); + if (b == -1) { + throw new EOFException("Reading Vulong past EOF"); + } + result |= (0x7f & b) << offset; + offset += 7; + } while (b >= 0x80); + return result; + } + + static long readVslong(InputStream in) throws IOException { + long result = readVulong(in); + return (result >>> 1) ^ -(result & 1); + } + + static float readFloat(InputStream in) throws IOException { + int ser = in.read() | (in.read() << 8) | (in.read() << 16) | + (in.read() << 24); + return Float.intBitsToFloat(ser); + } + + static void writeFloat(OutputStream output, float value) throws IOException { + int ser = Float.floatToIntBits(value); + output.write(ser & 0xff); + output.write((ser >> 8) & 0xff); + output.write((ser >> 16) & 0xff); + output.write((ser >> 24) & 0xff); + } + + static double readDouble(InputStream in) throws IOException { + long ser = in.read() | (in.read() << 8) | (in.read() << 16) | + (in.read() << 24) | ((long) in.read() << 32) | ((long) in.read() << 40) | + ((long) in.read() << 48) | ((long) in.read() << 56); + return Double.longBitsToDouble(ser); + } + + static void writeDouble(OutputStream output, + double value) throws IOException { + long ser = Double.doubleToLongBits(value); + output.write(((int) ser) & 0xff); + output.write(((int) (ser >> 8)) & 0xff); + output.write(((int) (ser >> 16)) & 0xff); + output.write(((int) (ser >> 24)) & 0xff); + output.write(((int) (ser >> 32)) & 0xff); + output.write(((int) (ser >> 40)) & 0xff); + output.write(((int) (ser >> 48)) & 0xff); + output.write(((int) (ser >> 56)) & 0xff); + } +} Index: contrib/src/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java =================================================================== --- contrib/src/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java (revision 0) +++ contrib/src/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java (working copy) @@ -0,0 +1,9231 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: contrib/if/orc_proto.proto + +package org.apache.hadoop.hive.ql.io.orc; + +public final class OrcProto { + private OrcProto() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public enum ColumnEncoding + implements com.google.protobuf.ProtocolMessageEnum { + DIRECT(0, 0), + DICTIONARY(1, 1), + ; + + public static final int DIRECT_VALUE = 0; + public static final int DICTIONARY_VALUE = 1; + + + public final int getNumber() { return value; } + + public static ColumnEncoding valueOf(int value) { + switch (value) { + case 0: return DIRECT; + case 1: return DICTIONARY; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public ColumnEncoding findValueByNumber(int number) { + return ColumnEncoding.valueOf(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(index); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.getDescriptor().getEnumTypes().get(0); + } + + private static final ColumnEncoding[] VALUES = { + DIRECT, DICTIONARY, + }; + + public static ColumnEncoding valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int index; + private final int value; + + private ColumnEncoding(int index, int value) { + this.index = index; + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:org.apache.hadoop.hive.ql.io.orc.ColumnEncoding) + } + + public enum CompressionKind + implements com.google.protobuf.ProtocolMessageEnum { + NONE(0, 0), + ZLIB(1, 1), + SNAPPY(2, 2), + LZO(3, 3), + ; + + public static final int NONE_VALUE = 0; + public static final int ZLIB_VALUE = 1; + public static final int SNAPPY_VALUE = 2; + public static final int LZO_VALUE = 3; + + + public final int getNumber() { return value; } + + public static CompressionKind valueOf(int value) { + switch (value) { + case 0: return NONE; + case 1: return ZLIB; + case 2: return SNAPPY; + case 3: return LZO; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public CompressionKind findValueByNumber(int number) { + return CompressionKind.valueOf(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(index); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.getDescriptor().getEnumTypes().get(1); + } + + private static final CompressionKind[] VALUES = { + NONE, ZLIB, SNAPPY, LZO, + }; + + public static CompressionKind valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int index; + private final int value; + + private CompressionKind(int index, int value) { + this.index = index; + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:org.apache.hadoop.hive.ql.io.orc.CompressionKind) + } + + public interface IntegerStatisticsOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional sint64 minimum = 1; + boolean hasMinimum(); + long getMinimum(); + + // optional sint64 maximum = 2; + boolean hasMaximum(); + long getMaximum(); + + // optional sint64 sum = 3; + boolean hasSum(); + long getSum(); + } + public static final class IntegerStatistics extends + com.google.protobuf.GeneratedMessage + implements IntegerStatisticsOrBuilder { + // Use IntegerStatistics.newBuilder() to construct. + private IntegerStatistics(Builder builder) { + super(builder); + } + private IntegerStatistics(boolean noInit) {} + + private static final IntegerStatistics defaultInstance; + public static IntegerStatistics getDefaultInstance() { + return defaultInstance; + } + + public IntegerStatistics getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_IntegerStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_IntegerStatistics_fieldAccessorTable; + } + + private int bitField0_; + // optional sint64 minimum = 1; + public static final int MINIMUM_FIELD_NUMBER = 1; + private long minimum_; + public boolean hasMinimum() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getMinimum() { + return minimum_; + } + + // optional sint64 maximum = 2; + public static final int MAXIMUM_FIELD_NUMBER = 2; + private long maximum_; + public boolean hasMaximum() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getMaximum() { + return maximum_; + } + + // optional sint64 sum = 3; + public static final int SUM_FIELD_NUMBER = 3; + private long sum_; + public boolean hasSum() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getSum() { + return sum_; + } + + private void initFields() { + minimum_ = 0L; + maximum_ = 0L; + sum_ = 0L; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeSInt64(1, minimum_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeSInt64(2, maximum_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeSInt64(3, sum_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeSInt64Size(1, minimum_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeSInt64Size(2, maximum_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeSInt64Size(3, sum_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatisticsOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_IntegerStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_IntegerStatistics_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + minimum_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + maximum_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + sum_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.minimum_ = minimum_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.maximum_ = maximum_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.sum_ = sum_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance()) return this; + if (other.hasMinimum()) { + setMinimum(other.getMinimum()); + } + if (other.hasMaximum()) { + setMaximum(other.getMaximum()); + } + if (other.hasSum()) { + setSum(other.getSum()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + minimum_ = input.readSInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + maximum_ = input.readSInt64(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + sum_ = input.readSInt64(); + break; + } + } + } + } + + private int bitField0_; + + // optional sint64 minimum = 1; + private long minimum_ ; + public boolean hasMinimum() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getMinimum() { + return minimum_; + } + public Builder setMinimum(long value) { + bitField0_ |= 0x00000001; + minimum_ = value; + onChanged(); + return this; + } + public Builder clearMinimum() { + bitField0_ = (bitField0_ & ~0x00000001); + minimum_ = 0L; + onChanged(); + return this; + } + + // optional sint64 maximum = 2; + private long maximum_ ; + public boolean hasMaximum() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getMaximum() { + return maximum_; + } + public Builder setMaximum(long value) { + bitField0_ |= 0x00000002; + maximum_ = value; + onChanged(); + return this; + } + public Builder clearMaximum() { + bitField0_ = (bitField0_ & ~0x00000002); + maximum_ = 0L; + onChanged(); + return this; + } + + // optional sint64 sum = 3; + private long sum_ ; + public boolean hasSum() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getSum() { + return sum_; + } + public Builder setSum(long value) { + bitField0_ |= 0x00000004; + sum_ = value; + onChanged(); + return this; + } + public Builder clearSum() { + bitField0_ = (bitField0_ & ~0x00000004); + sum_ = 0L; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.IntegerStatistics) + } + + static { + defaultInstance = new IntegerStatistics(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.IntegerStatistics) + } + + public interface DoubleStatisticsOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional double minimum = 1; + boolean hasMinimum(); + double getMinimum(); + + // optional double maximum = 2; + boolean hasMaximum(); + double getMaximum(); + + // optional double sum = 3; + boolean hasSum(); + double getSum(); + } + public static final class DoubleStatistics extends + com.google.protobuf.GeneratedMessage + implements DoubleStatisticsOrBuilder { + // Use DoubleStatistics.newBuilder() to construct. + private DoubleStatistics(Builder builder) { + super(builder); + } + private DoubleStatistics(boolean noInit) {} + + private static final DoubleStatistics defaultInstance; + public static DoubleStatistics getDefaultInstance() { + return defaultInstance; + } + + public DoubleStatistics getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_DoubleStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_DoubleStatistics_fieldAccessorTable; + } + + private int bitField0_; + // optional double minimum = 1; + public static final int MINIMUM_FIELD_NUMBER = 1; + private double minimum_; + public boolean hasMinimum() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public double getMinimum() { + return minimum_; + } + + // optional double maximum = 2; + public static final int MAXIMUM_FIELD_NUMBER = 2; + private double maximum_; + public boolean hasMaximum() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public double getMaximum() { + return maximum_; + } + + // optional double sum = 3; + public static final int SUM_FIELD_NUMBER = 3; + private double sum_; + public boolean hasSum() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public double getSum() { + return sum_; + } + + private void initFields() { + minimum_ = 0D; + maximum_ = 0D; + sum_ = 0D; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeDouble(1, minimum_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeDouble(2, maximum_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeDouble(3, sum_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeDoubleSize(1, minimum_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeDoubleSize(2, maximum_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeDoubleSize(3, sum_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatisticsOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_DoubleStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_DoubleStatistics_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + minimum_ = 0D; + bitField0_ = (bitField0_ & ~0x00000001); + maximum_ = 0D; + bitField0_ = (bitField0_ & ~0x00000002); + sum_ = 0D; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.minimum_ = minimum_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.maximum_ = maximum_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.sum_ = sum_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDefaultInstance()) return this; + if (other.hasMinimum()) { + setMinimum(other.getMinimum()); + } + if (other.hasMaximum()) { + setMaximum(other.getMaximum()); + } + if (other.hasSum()) { + setSum(other.getSum()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 9: { + bitField0_ |= 0x00000001; + minimum_ = input.readDouble(); + break; + } + case 17: { + bitField0_ |= 0x00000002; + maximum_ = input.readDouble(); + break; + } + case 25: { + bitField0_ |= 0x00000004; + sum_ = input.readDouble(); + break; + } + } + } + } + + private int bitField0_; + + // optional double minimum = 1; + private double minimum_ ; + public boolean hasMinimum() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public double getMinimum() { + return minimum_; + } + public Builder setMinimum(double value) { + bitField0_ |= 0x00000001; + minimum_ = value; + onChanged(); + return this; + } + public Builder clearMinimum() { + bitField0_ = (bitField0_ & ~0x00000001); + minimum_ = 0D; + onChanged(); + return this; + } + + // optional double maximum = 2; + private double maximum_ ; + public boolean hasMaximum() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public double getMaximum() { + return maximum_; + } + public Builder setMaximum(double value) { + bitField0_ |= 0x00000002; + maximum_ = value; + onChanged(); + return this; + } + public Builder clearMaximum() { + bitField0_ = (bitField0_ & ~0x00000002); + maximum_ = 0D; + onChanged(); + return this; + } + + // optional double sum = 3; + private double sum_ ; + public boolean hasSum() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public double getSum() { + return sum_; + } + public Builder setSum(double value) { + bitField0_ |= 0x00000004; + sum_ = value; + onChanged(); + return this; + } + public Builder clearSum() { + bitField0_ = (bitField0_ & ~0x00000004); + sum_ = 0D; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.DoubleStatistics) + } + + static { + defaultInstance = new DoubleStatistics(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.DoubleStatistics) + } + + public interface StringStatisticsOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional string minimum = 1; + boolean hasMinimum(); + String getMinimum(); + + // optional string maximum = 2; + boolean hasMaximum(); + String getMaximum(); + } + public static final class StringStatistics extends + com.google.protobuf.GeneratedMessage + implements StringStatisticsOrBuilder { + // Use StringStatistics.newBuilder() to construct. + private StringStatistics(Builder builder) { + super(builder); + } + private StringStatistics(boolean noInit) {} + + private static final StringStatistics defaultInstance; + public static StringStatistics getDefaultInstance() { + return defaultInstance; + } + + public StringStatistics getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StringStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StringStatistics_fieldAccessorTable; + } + + private int bitField0_; + // optional string minimum = 1; + public static final int MINIMUM_FIELD_NUMBER = 1; + private java.lang.Object minimum_; + public boolean hasMinimum() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getMinimum() { + java.lang.Object ref = minimum_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + minimum_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getMinimumBytes() { + java.lang.Object ref = minimum_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + minimum_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string maximum = 2; + public static final int MAXIMUM_FIELD_NUMBER = 2; + private java.lang.Object maximum_; + public boolean hasMaximum() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getMaximum() { + java.lang.Object ref = maximum_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + maximum_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getMaximumBytes() { + java.lang.Object ref = maximum_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + maximum_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + minimum_ = ""; + maximum_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getMinimumBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, getMaximumBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getMinimumBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, getMaximumBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatisticsOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StringStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StringStatistics_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + minimum_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + maximum_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.minimum_ = minimum_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.maximum_ = maximum_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance()) return this; + if (other.hasMinimum()) { + setMinimum(other.getMinimum()); + } + if (other.hasMaximum()) { + setMaximum(other.getMaximum()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + minimum_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + maximum_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // optional string minimum = 1; + private java.lang.Object minimum_ = ""; + public boolean hasMinimum() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getMinimum() { + java.lang.Object ref = minimum_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + minimum_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setMinimum(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + minimum_ = value; + onChanged(); + return this; + } + public Builder clearMinimum() { + bitField0_ = (bitField0_ & ~0x00000001); + minimum_ = getDefaultInstance().getMinimum(); + onChanged(); + return this; + } + void setMinimum(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + minimum_ = value; + onChanged(); + } + + // optional string maximum = 2; + private java.lang.Object maximum_ = ""; + public boolean hasMaximum() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getMaximum() { + java.lang.Object ref = maximum_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + maximum_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setMaximum(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + maximum_ = value; + onChanged(); + return this; + } + public Builder clearMaximum() { + bitField0_ = (bitField0_ & ~0x00000002); + maximum_ = getDefaultInstance().getMaximum(); + onChanged(); + return this; + } + void setMaximum(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000002; + maximum_ = value; + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.StringStatistics) + } + + static { + defaultInstance = new StringStatistics(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.StringStatistics) + } + + public interface BucketStatisticsOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated uint64 count = 1 [packed = true]; + java.util.List getCountList(); + int getCountCount(); + long getCount(int index); + } + public static final class BucketStatistics extends + com.google.protobuf.GeneratedMessage + implements BucketStatisticsOrBuilder { + // Use BucketStatistics.newBuilder() to construct. + private BucketStatistics(Builder builder) { + super(builder); + } + private BucketStatistics(boolean noInit) {} + + private static final BucketStatistics defaultInstance; + public static BucketStatistics getDefaultInstance() { + return defaultInstance; + } + + public BucketStatistics getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_BucketStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_BucketStatistics_fieldAccessorTable; + } + + // repeated uint64 count = 1 [packed = true]; + public static final int COUNT_FIELD_NUMBER = 1; + private java.util.List count_; + public java.util.List + getCountList() { + return count_; + } + public int getCountCount() { + return count_.size(); + } + public long getCount(int index) { + return count_.get(index); + } + private int countMemoizedSerializedSize = -1; + + private void initFields() { + count_ = java.util.Collections.emptyList();; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (getCountList().size() > 0) { + output.writeRawVarint32(10); + output.writeRawVarint32(countMemoizedSerializedSize); + } + for (int i = 0; i < count_.size(); i++) { + output.writeUInt64NoTag(count_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + { + int dataSize = 0; + for (int i = 0; i < count_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeUInt64SizeNoTag(count_.get(i)); + } + size += dataSize; + if (!getCountList().isEmpty()) { + size += 1; + size += com.google.protobuf.CodedOutputStream + .computeInt32SizeNoTag(dataSize); + } + countMemoizedSerializedSize = dataSize; + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatisticsOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_BucketStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_BucketStatistics_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + count_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics(this); + int from_bitField0_ = bitField0_; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + count_ = java.util.Collections.unmodifiableList(count_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.count_ = count_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance()) return this; + if (!other.count_.isEmpty()) { + if (count_.isEmpty()) { + count_ = other.count_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureCountIsMutable(); + count_.addAll(other.count_); + } + onChanged(); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + ensureCountIsMutable(); + count_.add(input.readUInt64()); + break; + } + case 10: { + int length = input.readRawVarint32(); + int limit = input.pushLimit(length); + while (input.getBytesUntilLimit() > 0) { + addCount(input.readUInt64()); + } + input.popLimit(limit); + break; + } + } + } + } + + private int bitField0_; + + // repeated uint64 count = 1 [packed = true]; + private java.util.List count_ = java.util.Collections.emptyList();; + private void ensureCountIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + count_ = new java.util.ArrayList(count_); + bitField0_ |= 0x00000001; + } + } + public java.util.List + getCountList() { + return java.util.Collections.unmodifiableList(count_); + } + public int getCountCount() { + return count_.size(); + } + public long getCount(int index) { + return count_.get(index); + } + public Builder setCount( + int index, long value) { + ensureCountIsMutable(); + count_.set(index, value); + onChanged(); + return this; + } + public Builder addCount(long value) { + ensureCountIsMutable(); + count_.add(value); + onChanged(); + return this; + } + public Builder addAllCount( + java.lang.Iterable values) { + ensureCountIsMutable(); + super.addAll(values, count_); + onChanged(); + return this; + } + public Builder clearCount() { + count_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.BucketStatistics) + } + + static { + defaultInstance = new BucketStatistics(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.BucketStatistics) + } + + public interface ColumnStatisticsOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional uint64 numberOfValues = 1; + boolean hasNumberOfValues(); + long getNumberOfValues(); + + // optional .org.apache.hadoop.hive.ql.io.orc.IntegerStatistics intStatistics = 2; + boolean hasIntStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics getIntStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatisticsOrBuilder getIntStatisticsOrBuilder(); + + // optional .org.apache.hadoop.hive.ql.io.orc.DoubleStatistics doubleStatistics = 3; + boolean hasDoubleStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics getDoubleStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatisticsOrBuilder getDoubleStatisticsOrBuilder(); + + // optional .org.apache.hadoop.hive.ql.io.orc.StringStatistics stringStatistics = 4; + boolean hasStringStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics getStringStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatisticsOrBuilder getStringStatisticsOrBuilder(); + + // optional .org.apache.hadoop.hive.ql.io.orc.BucketStatistics bucketStatistics = 5; + boolean hasBucketStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics getBucketStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatisticsOrBuilder getBucketStatisticsOrBuilder(); + } + public static final class ColumnStatistics extends + com.google.protobuf.GeneratedMessage + implements ColumnStatisticsOrBuilder { + // Use ColumnStatistics.newBuilder() to construct. + private ColumnStatistics(Builder builder) { + super(builder); + } + private ColumnStatistics(boolean noInit) {} + + private static final ColumnStatistics defaultInstance; + public static ColumnStatistics getDefaultInstance() { + return defaultInstance; + } + + public ColumnStatistics getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_fieldAccessorTable; + } + + private int bitField0_; + // optional uint64 numberOfValues = 1; + public static final int NUMBEROFVALUES_FIELD_NUMBER = 1; + private long numberOfValues_; + public boolean hasNumberOfValues() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getNumberOfValues() { + return numberOfValues_; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.IntegerStatistics intStatistics = 2; + public static final int INTSTATISTICS_FIELD_NUMBER = 2; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics intStatistics_; + public boolean hasIntStatistics() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics getIntStatistics() { + return intStatistics_; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatisticsOrBuilder getIntStatisticsOrBuilder() { + return intStatistics_; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.DoubleStatistics doubleStatistics = 3; + public static final int DOUBLESTATISTICS_FIELD_NUMBER = 3; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics doubleStatistics_; + public boolean hasDoubleStatistics() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics getDoubleStatistics() { + return doubleStatistics_; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatisticsOrBuilder getDoubleStatisticsOrBuilder() { + return doubleStatistics_; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.StringStatistics stringStatistics = 4; + public static final int STRINGSTATISTICS_FIELD_NUMBER = 4; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics stringStatistics_; + public boolean hasStringStatistics() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics getStringStatistics() { + return stringStatistics_; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatisticsOrBuilder getStringStatisticsOrBuilder() { + return stringStatistics_; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.BucketStatistics bucketStatistics = 5; + public static final int BUCKETSTATISTICS_FIELD_NUMBER = 5; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics bucketStatistics_; + public boolean hasBucketStatistics() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics getBucketStatistics() { + return bucketStatistics_; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatisticsOrBuilder getBucketStatisticsOrBuilder() { + return bucketStatistics_; + } + + private void initFields() { + numberOfValues_ = 0L; + intStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance(); + doubleStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDefaultInstance(); + stringStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance(); + bucketStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeUInt64(1, numberOfValues_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeMessage(2, intStatistics_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeMessage(3, doubleStatistics_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeMessage(4, stringStatistics_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeMessage(5, bucketStatistics_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(1, numberOfValues_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, intStatistics_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(3, doubleStatistics_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(4, stringStatistics_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(5, bucketStatistics_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getIntStatisticsFieldBuilder(); + getDoubleStatisticsFieldBuilder(); + getStringStatisticsFieldBuilder(); + getBucketStatisticsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + numberOfValues_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + if (intStatisticsBuilder_ == null) { + intStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance(); + } else { + intStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + if (doubleStatisticsBuilder_ == null) { + doubleStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDefaultInstance(); + } else { + doubleStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + if (stringStatisticsBuilder_ == null) { + stringStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance(); + } else { + stringStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000008); + if (bucketStatisticsBuilder_ == null) { + bucketStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance(); + } else { + bucketStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.numberOfValues_ = numberOfValues_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + if (intStatisticsBuilder_ == null) { + result.intStatistics_ = intStatistics_; + } else { + result.intStatistics_ = intStatisticsBuilder_.build(); + } + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + if (doubleStatisticsBuilder_ == null) { + result.doubleStatistics_ = doubleStatistics_; + } else { + result.doubleStatistics_ = doubleStatisticsBuilder_.build(); + } + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + if (stringStatisticsBuilder_ == null) { + result.stringStatistics_ = stringStatistics_; + } else { + result.stringStatistics_ = stringStatisticsBuilder_.build(); + } + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + if (bucketStatisticsBuilder_ == null) { + result.bucketStatistics_ = bucketStatistics_; + } else { + result.bucketStatistics_ = bucketStatisticsBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance()) return this; + if (other.hasNumberOfValues()) { + setNumberOfValues(other.getNumberOfValues()); + } + if (other.hasIntStatistics()) { + mergeIntStatistics(other.getIntStatistics()); + } + if (other.hasDoubleStatistics()) { + mergeDoubleStatistics(other.getDoubleStatistics()); + } + if (other.hasStringStatistics()) { + mergeStringStatistics(other.getStringStatistics()); + } + if (other.hasBucketStatistics()) { + mergeBucketStatistics(other.getBucketStatistics()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + numberOfValues_ = input.readUInt64(); + break; + } + case 18: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.newBuilder(); + if (hasIntStatistics()) { + subBuilder.mergeFrom(getIntStatistics()); + } + input.readMessage(subBuilder, extensionRegistry); + setIntStatistics(subBuilder.buildPartial()); + break; + } + case 26: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.newBuilder(); + if (hasDoubleStatistics()) { + subBuilder.mergeFrom(getDoubleStatistics()); + } + input.readMessage(subBuilder, extensionRegistry); + setDoubleStatistics(subBuilder.buildPartial()); + break; + } + case 34: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.newBuilder(); + if (hasStringStatistics()) { + subBuilder.mergeFrom(getStringStatistics()); + } + input.readMessage(subBuilder, extensionRegistry); + setStringStatistics(subBuilder.buildPartial()); + break; + } + case 42: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.newBuilder(); + if (hasBucketStatistics()) { + subBuilder.mergeFrom(getBucketStatistics()); + } + input.readMessage(subBuilder, extensionRegistry); + setBucketStatistics(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // optional uint64 numberOfValues = 1; + private long numberOfValues_ ; + public boolean hasNumberOfValues() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getNumberOfValues() { + return numberOfValues_; + } + public Builder setNumberOfValues(long value) { + bitField0_ |= 0x00000001; + numberOfValues_ = value; + onChanged(); + return this; + } + public Builder clearNumberOfValues() { + bitField0_ = (bitField0_ & ~0x00000001); + numberOfValues_ = 0L; + onChanged(); + return this; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.IntegerStatistics intStatistics = 2; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics intStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatisticsOrBuilder> intStatisticsBuilder_; + public boolean hasIntStatistics() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics getIntStatistics() { + if (intStatisticsBuilder_ == null) { + return intStatistics_; + } else { + return intStatisticsBuilder_.getMessage(); + } + } + public Builder setIntStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics value) { + if (intStatisticsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + intStatistics_ = value; + onChanged(); + } else { + intStatisticsBuilder_.setMessage(value); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder setIntStatistics( + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.Builder builderForValue) { + if (intStatisticsBuilder_ == null) { + intStatistics_ = builderForValue.build(); + onChanged(); + } else { + intStatisticsBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder mergeIntStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics value) { + if (intStatisticsBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002) && + intStatistics_ != org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance()) { + intStatistics_ = + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.newBuilder(intStatistics_).mergeFrom(value).buildPartial(); + } else { + intStatistics_ = value; + } + onChanged(); + } else { + intStatisticsBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder clearIntStatistics() { + if (intStatisticsBuilder_ == null) { + intStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.getDefaultInstance(); + onChanged(); + } else { + intStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.Builder getIntStatisticsBuilder() { + bitField0_ |= 0x00000002; + onChanged(); + return getIntStatisticsFieldBuilder().getBuilder(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatisticsOrBuilder getIntStatisticsOrBuilder() { + if (intStatisticsBuilder_ != null) { + return intStatisticsBuilder_.getMessageOrBuilder(); + } else { + return intStatistics_; + } + } + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatisticsOrBuilder> + getIntStatisticsFieldBuilder() { + if (intStatisticsBuilder_ == null) { + intStatisticsBuilder_ = new com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatisticsOrBuilder>( + intStatistics_, + getParentForChildren(), + isClean()); + intStatistics_ = null; + } + return intStatisticsBuilder_; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.DoubleStatistics doubleStatistics = 3; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics doubleStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatisticsOrBuilder> doubleStatisticsBuilder_; + public boolean hasDoubleStatistics() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics getDoubleStatistics() { + if (doubleStatisticsBuilder_ == null) { + return doubleStatistics_; + } else { + return doubleStatisticsBuilder_.getMessage(); + } + } + public Builder setDoubleStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics value) { + if (doubleStatisticsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + doubleStatistics_ = value; + onChanged(); + } else { + doubleStatisticsBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + public Builder setDoubleStatistics( + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.Builder builderForValue) { + if (doubleStatisticsBuilder_ == null) { + doubleStatistics_ = builderForValue.build(); + onChanged(); + } else { + doubleStatisticsBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + public Builder mergeDoubleStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics value) { + if (doubleStatisticsBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && + doubleStatistics_ != org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDefaultInstance()) { + doubleStatistics_ = + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.newBuilder(doubleStatistics_).mergeFrom(value).buildPartial(); + } else { + doubleStatistics_ = value; + } + onChanged(); + } else { + doubleStatisticsBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + public Builder clearDoubleStatistics() { + if (doubleStatisticsBuilder_ == null) { + doubleStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.getDefaultInstance(); + onChanged(); + } else { + doubleStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.Builder getDoubleStatisticsBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getDoubleStatisticsFieldBuilder().getBuilder(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatisticsOrBuilder getDoubleStatisticsOrBuilder() { + if (doubleStatisticsBuilder_ != null) { + return doubleStatisticsBuilder_.getMessageOrBuilder(); + } else { + return doubleStatistics_; + } + } + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatisticsOrBuilder> + getDoubleStatisticsFieldBuilder() { + if (doubleStatisticsBuilder_ == null) { + doubleStatisticsBuilder_ = new com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatisticsOrBuilder>( + doubleStatistics_, + getParentForChildren(), + isClean()); + doubleStatistics_ = null; + } + return doubleStatisticsBuilder_; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.StringStatistics stringStatistics = 4; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics stringStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatisticsOrBuilder> stringStatisticsBuilder_; + public boolean hasStringStatistics() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics getStringStatistics() { + if (stringStatisticsBuilder_ == null) { + return stringStatistics_; + } else { + return stringStatisticsBuilder_.getMessage(); + } + } + public Builder setStringStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics value) { + if (stringStatisticsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + stringStatistics_ = value; + onChanged(); + } else { + stringStatisticsBuilder_.setMessage(value); + } + bitField0_ |= 0x00000008; + return this; + } + public Builder setStringStatistics( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.Builder builderForValue) { + if (stringStatisticsBuilder_ == null) { + stringStatistics_ = builderForValue.build(); + onChanged(); + } else { + stringStatisticsBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000008; + return this; + } + public Builder mergeStringStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics value) { + if (stringStatisticsBuilder_ == null) { + if (((bitField0_ & 0x00000008) == 0x00000008) && + stringStatistics_ != org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance()) { + stringStatistics_ = + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.newBuilder(stringStatistics_).mergeFrom(value).buildPartial(); + } else { + stringStatistics_ = value; + } + onChanged(); + } else { + stringStatisticsBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000008; + return this; + } + public Builder clearStringStatistics() { + if (stringStatisticsBuilder_ == null) { + stringStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.getDefaultInstance(); + onChanged(); + } else { + stringStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.Builder getStringStatisticsBuilder() { + bitField0_ |= 0x00000008; + onChanged(); + return getStringStatisticsFieldBuilder().getBuilder(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatisticsOrBuilder getStringStatisticsOrBuilder() { + if (stringStatisticsBuilder_ != null) { + return stringStatisticsBuilder_.getMessageOrBuilder(); + } else { + return stringStatistics_; + } + } + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatisticsOrBuilder> + getStringStatisticsFieldBuilder() { + if (stringStatisticsBuilder_ == null) { + stringStatisticsBuilder_ = new com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatisticsOrBuilder>( + stringStatistics_, + getParentForChildren(), + isClean()); + stringStatistics_ = null; + } + return stringStatisticsBuilder_; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.BucketStatistics bucketStatistics = 5; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics bucketStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatisticsOrBuilder> bucketStatisticsBuilder_; + public boolean hasBucketStatistics() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics getBucketStatistics() { + if (bucketStatisticsBuilder_ == null) { + return bucketStatistics_; + } else { + return bucketStatisticsBuilder_.getMessage(); + } + } + public Builder setBucketStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics value) { + if (bucketStatisticsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + bucketStatistics_ = value; + onChanged(); + } else { + bucketStatisticsBuilder_.setMessage(value); + } + bitField0_ |= 0x00000010; + return this; + } + public Builder setBucketStatistics( + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.Builder builderForValue) { + if (bucketStatisticsBuilder_ == null) { + bucketStatistics_ = builderForValue.build(); + onChanged(); + } else { + bucketStatisticsBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000010; + return this; + } + public Builder mergeBucketStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics value) { + if (bucketStatisticsBuilder_ == null) { + if (((bitField0_ & 0x00000010) == 0x00000010) && + bucketStatistics_ != org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance()) { + bucketStatistics_ = + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.newBuilder(bucketStatistics_).mergeFrom(value).buildPartial(); + } else { + bucketStatistics_ = value; + } + onChanged(); + } else { + bucketStatisticsBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000010; + return this; + } + public Builder clearBucketStatistics() { + if (bucketStatisticsBuilder_ == null) { + bucketStatistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.getDefaultInstance(); + onChanged(); + } else { + bucketStatisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.Builder getBucketStatisticsBuilder() { + bitField0_ |= 0x00000010; + onChanged(); + return getBucketStatisticsFieldBuilder().getBuilder(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatisticsOrBuilder getBucketStatisticsOrBuilder() { + if (bucketStatisticsBuilder_ != null) { + return bucketStatisticsBuilder_.getMessageOrBuilder(); + } else { + return bucketStatistics_; + } + } + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatisticsOrBuilder> + getBucketStatisticsFieldBuilder() { + if (bucketStatisticsBuilder_ == null) { + bucketStatisticsBuilder_ = new com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatisticsOrBuilder>( + bucketStatistics_, + getParentForChildren(), + isClean()); + bucketStatistics_ = null; + } + return bucketStatisticsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.ColumnStatistics) + } + + static { + defaultInstance = new ColumnStatistics(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.ColumnStatistics) + } + + public interface RowIndexEntryOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated uint64 positions = 1 [packed = true]; + java.util.List getPositionsList(); + int getPositionsCount(); + long getPositions(int index); + + // optional .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics statistics = 2; + boolean hasStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getStatistics(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getStatisticsOrBuilder(); + } + public static final class RowIndexEntry extends + com.google.protobuf.GeneratedMessage + implements RowIndexEntryOrBuilder { + // Use RowIndexEntry.newBuilder() to construct. + private RowIndexEntry(Builder builder) { + super(builder); + } + private RowIndexEntry(boolean noInit) {} + + private static final RowIndexEntry defaultInstance; + public static RowIndexEntry getDefaultInstance() { + return defaultInstance; + } + + public RowIndexEntry getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_fieldAccessorTable; + } + + private int bitField0_; + // repeated uint64 positions = 1 [packed = true]; + public static final int POSITIONS_FIELD_NUMBER = 1; + private java.util.List positions_; + public java.util.List + getPositionsList() { + return positions_; + } + public int getPositionsCount() { + return positions_.size(); + } + public long getPositions(int index) { + return positions_.get(index); + } + private int positionsMemoizedSerializedSize = -1; + + // optional .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics statistics = 2; + public static final int STATISTICS_FIELD_NUMBER = 2; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics statistics_; + public boolean hasStatistics() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getStatistics() { + return statistics_; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getStatisticsOrBuilder() { + return statistics_; + } + + private void initFields() { + positions_ = java.util.Collections.emptyList();; + statistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (getPositionsList().size() > 0) { + output.writeRawVarint32(10); + output.writeRawVarint32(positionsMemoizedSerializedSize); + } + for (int i = 0; i < positions_.size(); i++) { + output.writeUInt64NoTag(positions_.get(i)); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(2, statistics_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + { + int dataSize = 0; + for (int i = 0; i < positions_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeUInt64SizeNoTag(positions_.get(i)); + } + size += dataSize; + if (!getPositionsList().isEmpty()) { + size += 1; + size += com.google.protobuf.CodedOutputStream + .computeInt32SizeNoTag(dataSize); + } + positionsMemoizedSerializedSize = dataSize; + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, statistics_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntryOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getStatisticsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + positions_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000001); + if (statisticsBuilder_ == null) { + statistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance(); + } else { + statisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + positions_ = java.util.Collections.unmodifiableList(positions_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.positions_ = positions_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000001; + } + if (statisticsBuilder_ == null) { + result.statistics_ = statistics_; + } else { + result.statistics_ = statisticsBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.getDefaultInstance()) return this; + if (!other.positions_.isEmpty()) { + if (positions_.isEmpty()) { + positions_ = other.positions_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensurePositionsIsMutable(); + positions_.addAll(other.positions_); + } + onChanged(); + } + if (other.hasStatistics()) { + mergeStatistics(other.getStatistics()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + ensurePositionsIsMutable(); + positions_.add(input.readUInt64()); + break; + } + case 10: { + int length = input.readRawVarint32(); + int limit = input.pushLimit(length); + while (input.getBytesUntilLimit() > 0) { + addPositions(input.readUInt64()); + } + input.popLimit(limit); + break; + } + case 18: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.newBuilder(); + if (hasStatistics()) { + subBuilder.mergeFrom(getStatistics()); + } + input.readMessage(subBuilder, extensionRegistry); + setStatistics(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // repeated uint64 positions = 1 [packed = true]; + private java.util.List positions_ = java.util.Collections.emptyList();; + private void ensurePositionsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + positions_ = new java.util.ArrayList(positions_); + bitField0_ |= 0x00000001; + } + } + public java.util.List + getPositionsList() { + return java.util.Collections.unmodifiableList(positions_); + } + public int getPositionsCount() { + return positions_.size(); + } + public long getPositions(int index) { + return positions_.get(index); + } + public Builder setPositions( + int index, long value) { + ensurePositionsIsMutable(); + positions_.set(index, value); + onChanged(); + return this; + } + public Builder addPositions(long value) { + ensurePositionsIsMutable(); + positions_.add(value); + onChanged(); + return this; + } + public Builder addAllPositions( + java.lang.Iterable values) { + ensurePositionsIsMutable(); + super.addAll(values, positions_); + onChanged(); + return this; + } + public Builder clearPositions() { + positions_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics statistics = 2; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics statistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder> statisticsBuilder_; + public boolean hasStatistics() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getStatistics() { + if (statisticsBuilder_ == null) { + return statistics_; + } else { + return statisticsBuilder_.getMessage(); + } + } + public Builder setStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (statisticsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + statistics_ = value; + onChanged(); + } else { + statisticsBuilder_.setMessage(value); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder setStatistics( + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (statisticsBuilder_ == null) { + statistics_ = builderForValue.build(); + onChanged(); + } else { + statisticsBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder mergeStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (statisticsBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002) && + statistics_ != org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance()) { + statistics_ = + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.newBuilder(statistics_).mergeFrom(value).buildPartial(); + } else { + statistics_ = value; + } + onChanged(); + } else { + statisticsBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000002; + return this; + } + public Builder clearStatistics() { + if (statisticsBuilder_ == null) { + statistics_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance(); + onChanged(); + } else { + statisticsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder getStatisticsBuilder() { + bitField0_ |= 0x00000002; + onChanged(); + return getStatisticsFieldBuilder().getBuilder(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getStatisticsOrBuilder() { + if (statisticsBuilder_ != null) { + return statisticsBuilder_.getMessageOrBuilder(); + } else { + return statistics_; + } + } + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder> + getStatisticsFieldBuilder() { + if (statisticsBuilder_ == null) { + statisticsBuilder_ = new com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder>( + statistics_, + getParentForChildren(), + isClean()); + statistics_ = null; + } + return statisticsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.RowIndexEntry) + } + + static { + defaultInstance = new RowIndexEntry(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.RowIndexEntry) + } + + public interface RowIndexOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .org.apache.hadoop.hive.ql.io.orc.RowIndexEntry entry = 1; + java.util.List + getEntryList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry getEntry(int index); + int getEntryCount(); + java.util.List + getEntryOrBuilderList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntryOrBuilder getEntryOrBuilder( + int index); + } + public static final class RowIndex extends + com.google.protobuf.GeneratedMessage + implements RowIndexOrBuilder { + // Use RowIndex.newBuilder() to construct. + private RowIndex(Builder builder) { + super(builder); + } + private RowIndex(boolean noInit) {} + + private static final RowIndex defaultInstance; + public static RowIndex getDefaultInstance() { + return defaultInstance; + } + + public RowIndex getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_fieldAccessorTable; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.RowIndexEntry entry = 1; + public static final int ENTRY_FIELD_NUMBER = 1; + private java.util.List entry_; + public java.util.List getEntryList() { + return entry_; + } + public java.util.List + getEntryOrBuilderList() { + return entry_; + } + public int getEntryCount() { + return entry_.size(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry getEntry(int index) { + return entry_.get(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntryOrBuilder getEntryOrBuilder( + int index) { + return entry_.get(index); + } + + private void initFields() { + entry_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < entry_.size(); i++) { + output.writeMessage(1, entry_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < entry_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, entry_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getEntryFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (entryBuilder_ == null) { + entry_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + entryBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex(this); + int from_bitField0_ = bitField0_; + if (entryBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + entry_ = java.util.Collections.unmodifiableList(entry_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.entry_ = entry_; + } else { + result.entry_ = entryBuilder_.build(); + } + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex.getDefaultInstance()) return this; + if (entryBuilder_ == null) { + if (!other.entry_.isEmpty()) { + if (entry_.isEmpty()) { + entry_ = other.entry_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureEntryIsMutable(); + entry_.addAll(other.entry_); + } + onChanged(); + } + } else { + if (!other.entry_.isEmpty()) { + if (entryBuilder_.isEmpty()) { + entryBuilder_.dispose(); + entryBuilder_ = null; + entry_ = other.entry_; + bitField0_ = (bitField0_ & ~0x00000001); + entryBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getEntryFieldBuilder() : null; + } else { + entryBuilder_.addAllMessages(other.entry_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addEntry(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // repeated .org.apache.hadoop.hive.ql.io.orc.RowIndexEntry entry = 1; + private java.util.List entry_ = + java.util.Collections.emptyList(); + private void ensureEntryIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + entry_ = new java.util.ArrayList(entry_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntryOrBuilder> entryBuilder_; + + public java.util.List getEntryList() { + if (entryBuilder_ == null) { + return java.util.Collections.unmodifiableList(entry_); + } else { + return entryBuilder_.getMessageList(); + } + } + public int getEntryCount() { + if (entryBuilder_ == null) { + return entry_.size(); + } else { + return entryBuilder_.getCount(); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry getEntry(int index) { + if (entryBuilder_ == null) { + return entry_.get(index); + } else { + return entryBuilder_.getMessage(index); + } + } + public Builder setEntry( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry value) { + if (entryBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureEntryIsMutable(); + entry_.set(index, value); + onChanged(); + } else { + entryBuilder_.setMessage(index, value); + } + return this; + } + public Builder setEntry( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder builderForValue) { + if (entryBuilder_ == null) { + ensureEntryIsMutable(); + entry_.set(index, builderForValue.build()); + onChanged(); + } else { + entryBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addEntry(org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry value) { + if (entryBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureEntryIsMutable(); + entry_.add(value); + onChanged(); + } else { + entryBuilder_.addMessage(value); + } + return this; + } + public Builder addEntry( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry value) { + if (entryBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureEntryIsMutable(); + entry_.add(index, value); + onChanged(); + } else { + entryBuilder_.addMessage(index, value); + } + return this; + } + public Builder addEntry( + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder builderForValue) { + if (entryBuilder_ == null) { + ensureEntryIsMutable(); + entry_.add(builderForValue.build()); + onChanged(); + } else { + entryBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addEntry( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder builderForValue) { + if (entryBuilder_ == null) { + ensureEntryIsMutable(); + entry_.add(index, builderForValue.build()); + onChanged(); + } else { + entryBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllEntry( + java.lang.Iterable values) { + if (entryBuilder_ == null) { + ensureEntryIsMutable(); + super.addAll(values, entry_); + onChanged(); + } else { + entryBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearEntry() { + if (entryBuilder_ == null) { + entry_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + entryBuilder_.clear(); + } + return this; + } + public Builder removeEntry(int index) { + if (entryBuilder_ == null) { + ensureEntryIsMutable(); + entry_.remove(index); + onChanged(); + } else { + entryBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder getEntryBuilder( + int index) { + return getEntryFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntryOrBuilder getEntryOrBuilder( + int index) { + if (entryBuilder_ == null) { + return entry_.get(index); } else { + return entryBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getEntryOrBuilderList() { + if (entryBuilder_ != null) { + return entryBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(entry_); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder addEntryBuilder() { + return getEntryFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.getDefaultInstance()); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder addEntryBuilder( + int index) { + return getEntryFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.getDefaultInstance()); + } + public java.util.List + getEntryBuilderList() { + return getEntryFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntryOrBuilder> + getEntryFieldBuilder() { + if (entryBuilder_ == null) { + entryBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntryOrBuilder>( + entry_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + entry_ = null; + } + return entryBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.RowIndex) + } + + static { + defaultInstance = new RowIndex(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.RowIndex) + } + + public interface StreamOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required .org.apache.hadoop.hive.ql.io.orc.Stream.Kind kind = 1; + boolean hasKind(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind getKind(); + + // optional uint32 column = 2; + boolean hasColumn(); + int getColumn(); + + // optional uint64 length = 3; + boolean hasLength(); + long getLength(); + } + public static final class Stream extends + com.google.protobuf.GeneratedMessage + implements StreamOrBuilder { + // Use Stream.newBuilder() to construct. + private Stream(Builder builder) { + super(builder); + } + private Stream(boolean noInit) {} + + private static final Stream defaultInstance; + public static Stream getDefaultInstance() { + return defaultInstance; + } + + public Stream getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_fieldAccessorTable; + } + + public enum Kind + implements com.google.protobuf.ProtocolMessageEnum { + PRESENT(0, 0), + DATA(1, 1), + LENGTH(2, 2), + DICTIONARY_DATA(3, 3), + DICTIONARY_COUNT(4, 4), + NANO_DATA(5, 5), + ROW_INDEX(6, 6), + ; + + public static final int PRESENT_VALUE = 0; + public static final int DATA_VALUE = 1; + public static final int LENGTH_VALUE = 2; + public static final int DICTIONARY_DATA_VALUE = 3; + public static final int DICTIONARY_COUNT_VALUE = 4; + public static final int NANO_DATA_VALUE = 5; + public static final int ROW_INDEX_VALUE = 6; + + + public final int getNumber() { return value; } + + public static Kind valueOf(int value) { + switch (value) { + case 0: return PRESENT; + case 1: return DATA; + case 2: return LENGTH; + case 3: return DICTIONARY_DATA; + case 4: return DICTIONARY_COUNT; + case 5: return NANO_DATA; + case 6: return ROW_INDEX; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public Kind findValueByNumber(int number) { + return Kind.valueOf(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(index); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.getDescriptor().getEnumTypes().get(0); + } + + private static final Kind[] VALUES = { + PRESENT, DATA, LENGTH, DICTIONARY_DATA, DICTIONARY_COUNT, NANO_DATA, ROW_INDEX, + }; + + public static Kind valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int index; + private final int value; + + private Kind(int index, int value) { + this.index = index; + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:org.apache.hadoop.hive.ql.io.orc.Stream.Kind) + } + + private int bitField0_; + // required .org.apache.hadoop.hive.ql.io.orc.Stream.Kind kind = 1; + public static final int KIND_FIELD_NUMBER = 1; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind kind_; + public boolean hasKind() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind getKind() { + return kind_; + } + + // optional uint32 column = 2; + public static final int COLUMN_FIELD_NUMBER = 2; + private int column_; + public boolean hasColumn() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public int getColumn() { + return column_; + } + + // optional uint64 length = 3; + public static final int LENGTH_FIELD_NUMBER = 3; + private long length_; + public boolean hasLength() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getLength() { + return length_; + } + + private void initFields() { + kind_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind.PRESENT; + column_ = 0; + length_ = 0L; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasKind()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeEnum(1, kind_.getNumber()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeUInt32(2, column_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeUInt64(3, length_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(1, kind_.getNumber()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt32Size(2, column_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(3, length_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.StreamOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + kind_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind.PRESENT; + bitField0_ = (bitField0_ & ~0x00000001); + column_ = 0; + bitField0_ = (bitField0_ & ~0x00000002); + length_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.kind_ = kind_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.column_ = column_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.length_ = length_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.getDefaultInstance()) return this; + if (other.hasKind()) { + setKind(other.getKind()); + } + if (other.hasColumn()) { + setColumn(other.getColumn()); + } + if (other.hasLength()) { + setLength(other.getLength()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasKind()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + int rawValue = input.readEnum(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind value = org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(1, rawValue); + } else { + bitField0_ |= 0x00000001; + kind_ = value; + } + break; + } + case 16: { + bitField0_ |= 0x00000002; + column_ = input.readUInt32(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + length_ = input.readUInt64(); + break; + } + } + } + } + + private int bitField0_; + + // required .org.apache.hadoop.hive.ql.io.orc.Stream.Kind kind = 1; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind kind_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind.PRESENT; + public boolean hasKind() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind getKind() { + return kind_; + } + public Builder setKind(org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + kind_ = value; + onChanged(); + return this; + } + public Builder clearKind() { + bitField0_ = (bitField0_ & ~0x00000001); + kind_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Kind.PRESENT; + onChanged(); + return this; + } + + // optional uint32 column = 2; + private int column_ ; + public boolean hasColumn() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public int getColumn() { + return column_; + } + public Builder setColumn(int value) { + bitField0_ |= 0x00000002; + column_ = value; + onChanged(); + return this; + } + public Builder clearColumn() { + bitField0_ = (bitField0_ & ~0x00000002); + column_ = 0; + onChanged(); + return this; + } + + // optional uint64 length = 3; + private long length_ ; + public boolean hasLength() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getLength() { + return length_; + } + public Builder setLength(long value) { + bitField0_ |= 0x00000004; + length_ = value; + onChanged(); + return this; + } + public Builder clearLength() { + bitField0_ = (bitField0_ & ~0x00000004); + length_ = 0L; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.Stream) + } + + static { + defaultInstance = new Stream(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.Stream) + } + + public interface StripeFooterOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .org.apache.hadoop.hive.ql.io.orc.Stream streams = 1; + java.util.List + getStreamsList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream getStreams(int index); + int getStreamsCount(); + java.util.List + getStreamsOrBuilderList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.StreamOrBuilder getStreamsOrBuilder( + int index); + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnEncoding columns = 2 [packed = true]; + java.util.List getColumnsList(); + int getColumnsCount(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding getColumns(int index); + } + public static final class StripeFooter extends + com.google.protobuf.GeneratedMessage + implements StripeFooterOrBuilder { + // Use StripeFooter.newBuilder() to construct. + private StripeFooter(Builder builder) { + super(builder); + } + private StripeFooter(boolean noInit) {} + + private static final StripeFooter defaultInstance; + public static StripeFooter getDefaultInstance() { + return defaultInstance; + } + + public StripeFooter getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_fieldAccessorTable; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.Stream streams = 1; + public static final int STREAMS_FIELD_NUMBER = 1; + private java.util.List streams_; + public java.util.List getStreamsList() { + return streams_; + } + public java.util.List + getStreamsOrBuilderList() { + return streams_; + } + public int getStreamsCount() { + return streams_.size(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream getStreams(int index) { + return streams_.get(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StreamOrBuilder getStreamsOrBuilder( + int index) { + return streams_.get(index); + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnEncoding columns = 2 [packed = true]; + public static final int COLUMNS_FIELD_NUMBER = 2; + private java.util.List columns_; + public java.util.List getColumnsList() { + return columns_; + } + public int getColumnsCount() { + return columns_.size(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding getColumns(int index) { + return columns_.get(index); + } + private int columnsMemoizedSerializedSize; + + private void initFields() { + streams_ = java.util.Collections.emptyList(); + columns_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + for (int i = 0; i < getStreamsCount(); i++) { + if (!getStreams(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < streams_.size(); i++) { + output.writeMessage(1, streams_.get(i)); + } + if (getColumnsList().size() > 0) { + output.writeRawVarint32(18); + output.writeRawVarint32(columnsMemoizedSerializedSize); + } + for (int i = 0; i < columns_.size(); i++) { + output.writeEnumNoTag(columns_.get(i).getNumber()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < streams_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, streams_.get(i)); + } + { + int dataSize = 0; + for (int i = 0; i < columns_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeEnumSizeNoTag(columns_.get(i).getNumber()); + } + size += dataSize; + if (!getColumnsList().isEmpty()) { size += 1; + size += com.google.protobuf.CodedOutputStream + .computeRawVarint32Size(dataSize); + }columnsMemoizedSerializedSize = dataSize; + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooterOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getStreamsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (streamsBuilder_ == null) { + streams_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + streamsBuilder_.clear(); + } + columns_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter(this); + int from_bitField0_ = bitField0_; + if (streamsBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + streams_ = java.util.Collections.unmodifiableList(streams_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.streams_ = streams_; + } else { + result.streams_ = streamsBuilder_.build(); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + columns_ = java.util.Collections.unmodifiableList(columns_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.columns_ = columns_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter.getDefaultInstance()) return this; + if (streamsBuilder_ == null) { + if (!other.streams_.isEmpty()) { + if (streams_.isEmpty()) { + streams_ = other.streams_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureStreamsIsMutable(); + streams_.addAll(other.streams_); + } + onChanged(); + } + } else { + if (!other.streams_.isEmpty()) { + if (streamsBuilder_.isEmpty()) { + streamsBuilder_.dispose(); + streamsBuilder_ = null; + streams_ = other.streams_; + bitField0_ = (bitField0_ & ~0x00000001); + streamsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getStreamsFieldBuilder() : null; + } else { + streamsBuilder_.addAllMessages(other.streams_); + } + } + } + if (!other.columns_.isEmpty()) { + if (columns_.isEmpty()) { + columns_ = other.columns_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureColumnsIsMutable(); + columns_.addAll(other.columns_); + } + onChanged(); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + for (int i = 0; i < getStreamsCount(); i++) { + if (!getStreams(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addStreams(subBuilder.buildPartial()); + break; + } + case 16: { + int rawValue = input.readEnum(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding value = org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(2, rawValue); + } else { + addColumns(value); + } + break; + } + case 18: { + int length = input.readRawVarint32(); + int oldLimit = input.pushLimit(length); + while(input.getBytesUntilLimit() > 0) { + int rawValue = input.readEnum(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding value = org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(2, rawValue); + } else { + addColumns(value); + } + } + input.popLimit(oldLimit); + break; + } + } + } + } + + private int bitField0_; + + // repeated .org.apache.hadoop.hive.ql.io.orc.Stream streams = 1; + private java.util.List streams_ = + java.util.Collections.emptyList(); + private void ensureStreamsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + streams_ = new java.util.ArrayList(streams_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream, org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StreamOrBuilder> streamsBuilder_; + + public java.util.List getStreamsList() { + if (streamsBuilder_ == null) { + return java.util.Collections.unmodifiableList(streams_); + } else { + return streamsBuilder_.getMessageList(); + } + } + public int getStreamsCount() { + if (streamsBuilder_ == null) { + return streams_.size(); + } else { + return streamsBuilder_.getCount(); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream getStreams(int index) { + if (streamsBuilder_ == null) { + return streams_.get(index); + } else { + return streamsBuilder_.getMessage(index); + } + } + public Builder setStreams( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream value) { + if (streamsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStreamsIsMutable(); + streams_.set(index, value); + onChanged(); + } else { + streamsBuilder_.setMessage(index, value); + } + return this; + } + public Builder setStreams( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder builderForValue) { + if (streamsBuilder_ == null) { + ensureStreamsIsMutable(); + streams_.set(index, builderForValue.build()); + onChanged(); + } else { + streamsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addStreams(org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream value) { + if (streamsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStreamsIsMutable(); + streams_.add(value); + onChanged(); + } else { + streamsBuilder_.addMessage(value); + } + return this; + } + public Builder addStreams( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream value) { + if (streamsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStreamsIsMutable(); + streams_.add(index, value); + onChanged(); + } else { + streamsBuilder_.addMessage(index, value); + } + return this; + } + public Builder addStreams( + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder builderForValue) { + if (streamsBuilder_ == null) { + ensureStreamsIsMutable(); + streams_.add(builderForValue.build()); + onChanged(); + } else { + streamsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addStreams( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder builderForValue) { + if (streamsBuilder_ == null) { + ensureStreamsIsMutable(); + streams_.add(index, builderForValue.build()); + onChanged(); + } else { + streamsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllStreams( + java.lang.Iterable values) { + if (streamsBuilder_ == null) { + ensureStreamsIsMutable(); + super.addAll(values, streams_); + onChanged(); + } else { + streamsBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearStreams() { + if (streamsBuilder_ == null) { + streams_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + streamsBuilder_.clear(); + } + return this; + } + public Builder removeStreams(int index) { + if (streamsBuilder_ == null) { + ensureStreamsIsMutable(); + streams_.remove(index); + onChanged(); + } else { + streamsBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder getStreamsBuilder( + int index) { + return getStreamsFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StreamOrBuilder getStreamsOrBuilder( + int index) { + if (streamsBuilder_ == null) { + return streams_.get(index); } else { + return streamsBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getStreamsOrBuilderList() { + if (streamsBuilder_ != null) { + return streamsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(streams_); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder addStreamsBuilder() { + return getStreamsFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.getDefaultInstance()); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder addStreamsBuilder( + int index) { + return getStreamsFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.getDefaultInstance()); + } + public java.util.List + getStreamsBuilderList() { + return getStreamsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream, org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StreamOrBuilder> + getStreamsFieldBuilder() { + if (streamsBuilder_ == null) { + streamsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream, org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StreamOrBuilder>( + streams_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + streams_ = null; + } + return streamsBuilder_; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnEncoding columns = 2 [packed = true]; + private java.util.List columns_ = + java.util.Collections.emptyList(); + private void ensureColumnsIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + columns_ = new java.util.ArrayList(columns_); + bitField0_ |= 0x00000002; + } + } + public java.util.List getColumnsList() { + return java.util.Collections.unmodifiableList(columns_); + } + public int getColumnsCount() { + return columns_.size(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding getColumns(int index) { + return columns_.get(index); + } + public Builder setColumns( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding value) { + if (value == null) { + throw new NullPointerException(); + } + ensureColumnsIsMutable(); + columns_.set(index, value); + onChanged(); + return this; + } + public Builder addColumns(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnEncoding value) { + if (value == null) { + throw new NullPointerException(); + } + ensureColumnsIsMutable(); + columns_.add(value); + onChanged(); + return this; + } + public Builder addAllColumns( + java.lang.Iterable values) { + ensureColumnsIsMutable(); + super.addAll(values, columns_); + onChanged(); + return this; + } + public Builder clearColumns() { + columns_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.StripeFooter) + } + + static { + defaultInstance = new StripeFooter(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.StripeFooter) + } + + public interface TypeOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required .org.apache.hadoop.hive.ql.io.orc.Type.Kind kind = 1; + boolean hasKind(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind getKind(); + + // repeated uint32 subtypes = 2 [packed = true]; + java.util.List getSubtypesList(); + int getSubtypesCount(); + int getSubtypes(int index); + + // repeated string fieldNames = 3; + java.util.List getFieldNamesList(); + int getFieldNamesCount(); + String getFieldNames(int index); + } + public static final class Type extends + com.google.protobuf.GeneratedMessage + implements TypeOrBuilder { + // Use Type.newBuilder() to construct. + private Type(Builder builder) { + super(builder); + } + private Type(boolean noInit) {} + + private static final Type defaultInstance; + public static Type getDefaultInstance() { + return defaultInstance; + } + + public Type getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Type_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Type_fieldAccessorTable; + } + + public enum Kind + implements com.google.protobuf.ProtocolMessageEnum { + BOOLEAN(0, 0), + BYTE(1, 1), + SHORT(2, 2), + INT(3, 3), + LONG(4, 4), + FLOAT(5, 5), + DOUBLE(6, 6), + STRING(7, 7), + BINARY(8, 8), + TIMESTAMP(9, 9), + LIST(10, 10), + MAP(11, 11), + STRUCT(12, 12), + UNION(13, 13), + ; + + public static final int BOOLEAN_VALUE = 0; + public static final int BYTE_VALUE = 1; + public static final int SHORT_VALUE = 2; + public static final int INT_VALUE = 3; + public static final int LONG_VALUE = 4; + public static final int FLOAT_VALUE = 5; + public static final int DOUBLE_VALUE = 6; + public static final int STRING_VALUE = 7; + public static final int BINARY_VALUE = 8; + public static final int TIMESTAMP_VALUE = 9; + public static final int LIST_VALUE = 10; + public static final int MAP_VALUE = 11; + public static final int STRUCT_VALUE = 12; + public static final int UNION_VALUE = 13; + + + public final int getNumber() { return value; } + + public static Kind valueOf(int value) { + switch (value) { + case 0: return BOOLEAN; + case 1: return BYTE; + case 2: return SHORT; + case 3: return INT; + case 4: return LONG; + case 5: return FLOAT; + case 6: return DOUBLE; + case 7: return STRING; + case 8: return BINARY; + case 9: return TIMESTAMP; + case 10: return LIST; + case 11: return MAP; + case 12: return STRUCT; + case 13: return UNION; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public Kind findValueByNumber(int number) { + return Kind.valueOf(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(index); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.getDescriptor().getEnumTypes().get(0); + } + + private static final Kind[] VALUES = { + BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, BINARY, TIMESTAMP, LIST, MAP, STRUCT, UNION, + }; + + public static Kind valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int index; + private final int value; + + private Kind(int index, int value) { + this.index = index; + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:org.apache.hadoop.hive.ql.io.orc.Type.Kind) + } + + private int bitField0_; + // required .org.apache.hadoop.hive.ql.io.orc.Type.Kind kind = 1; + public static final int KIND_FIELD_NUMBER = 1; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind kind_; + public boolean hasKind() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind getKind() { + return kind_; + } + + // repeated uint32 subtypes = 2 [packed = true]; + public static final int SUBTYPES_FIELD_NUMBER = 2; + private java.util.List subtypes_; + public java.util.List + getSubtypesList() { + return subtypes_; + } + public int getSubtypesCount() { + return subtypes_.size(); + } + public int getSubtypes(int index) { + return subtypes_.get(index); + } + private int subtypesMemoizedSerializedSize = -1; + + // repeated string fieldNames = 3; + public static final int FIELDNAMES_FIELD_NUMBER = 3; + private com.google.protobuf.LazyStringList fieldNames_; + public java.util.List + getFieldNamesList() { + return fieldNames_; + } + public int getFieldNamesCount() { + return fieldNames_.size(); + } + public String getFieldNames(int index) { + return fieldNames_.get(index); + } + + private void initFields() { + kind_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind.BOOLEAN; + subtypes_ = java.util.Collections.emptyList();; + fieldNames_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasKind()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeEnum(1, kind_.getNumber()); + } + if (getSubtypesList().size() > 0) { + output.writeRawVarint32(18); + output.writeRawVarint32(subtypesMemoizedSerializedSize); + } + for (int i = 0; i < subtypes_.size(); i++) { + output.writeUInt32NoTag(subtypes_.get(i)); + } + for (int i = 0; i < fieldNames_.size(); i++) { + output.writeBytes(3, fieldNames_.getByteString(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(1, kind_.getNumber()); + } + { + int dataSize = 0; + for (int i = 0; i < subtypes_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeUInt32SizeNoTag(subtypes_.get(i)); + } + size += dataSize; + if (!getSubtypesList().isEmpty()) { + size += 1; + size += com.google.protobuf.CodedOutputStream + .computeInt32SizeNoTag(dataSize); + } + subtypesMemoizedSerializedSize = dataSize; + } + { + int dataSize = 0; + for (int i = 0; i < fieldNames_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeBytesSizeNoTag(fieldNames_.getByteString(i)); + } + size += dataSize; + size += 1 * getFieldNamesList().size(); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Type parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.Type prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.TypeOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Type_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Type_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + kind_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind.BOOLEAN; + bitField0_ = (bitField0_ & ~0x00000001); + subtypes_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000002); + fieldNames_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.Type buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.Type(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.kind_ = kind_; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + subtypes_ = java.util.Collections.unmodifiableList(subtypes_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.subtypes_ = subtypes_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + fieldNames_ = new com.google.protobuf.UnmodifiableLazyStringList( + fieldNames_); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.fieldNames_ = fieldNames_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.Type) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.Type)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.Type other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.getDefaultInstance()) return this; + if (other.hasKind()) { + setKind(other.getKind()); + } + if (!other.subtypes_.isEmpty()) { + if (subtypes_.isEmpty()) { + subtypes_ = other.subtypes_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureSubtypesIsMutable(); + subtypes_.addAll(other.subtypes_); + } + onChanged(); + } + if (!other.fieldNames_.isEmpty()) { + if (fieldNames_.isEmpty()) { + fieldNames_ = other.fieldNames_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureFieldNamesIsMutable(); + fieldNames_.addAll(other.fieldNames_); + } + onChanged(); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasKind()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + int rawValue = input.readEnum(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind value = org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(1, rawValue); + } else { + bitField0_ |= 0x00000001; + kind_ = value; + } + break; + } + case 16: { + ensureSubtypesIsMutable(); + subtypes_.add(input.readUInt32()); + break; + } + case 18: { + int length = input.readRawVarint32(); + int limit = input.pushLimit(length); + while (input.getBytesUntilLimit() > 0) { + addSubtypes(input.readUInt32()); + } + input.popLimit(limit); + break; + } + case 26: { + ensureFieldNamesIsMutable(); + fieldNames_.add(input.readBytes()); + break; + } + } + } + } + + private int bitField0_; + + // required .org.apache.hadoop.hive.ql.io.orc.Type.Kind kind = 1; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind kind_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind.BOOLEAN; + public boolean hasKind() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind getKind() { + return kind_; + } + public Builder setKind(org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + kind_ = value; + onChanged(); + return this; + } + public Builder clearKind() { + bitField0_ = (bitField0_ & ~0x00000001); + kind_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Kind.BOOLEAN; + onChanged(); + return this; + } + + // repeated uint32 subtypes = 2 [packed = true]; + private java.util.List subtypes_ = java.util.Collections.emptyList();; + private void ensureSubtypesIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + subtypes_ = new java.util.ArrayList(subtypes_); + bitField0_ |= 0x00000002; + } + } + public java.util.List + getSubtypesList() { + return java.util.Collections.unmodifiableList(subtypes_); + } + public int getSubtypesCount() { + return subtypes_.size(); + } + public int getSubtypes(int index) { + return subtypes_.get(index); + } + public Builder setSubtypes( + int index, int value) { + ensureSubtypesIsMutable(); + subtypes_.set(index, value); + onChanged(); + return this; + } + public Builder addSubtypes(int value) { + ensureSubtypesIsMutable(); + subtypes_.add(value); + onChanged(); + return this; + } + public Builder addAllSubtypes( + java.lang.Iterable values) { + ensureSubtypesIsMutable(); + super.addAll(values, subtypes_); + onChanged(); + return this; + } + public Builder clearSubtypes() { + subtypes_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + return this; + } + + // repeated string fieldNames = 3; + private com.google.protobuf.LazyStringList fieldNames_ = com.google.protobuf.LazyStringArrayList.EMPTY; + private void ensureFieldNamesIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + fieldNames_ = new com.google.protobuf.LazyStringArrayList(fieldNames_); + bitField0_ |= 0x00000004; + } + } + public java.util.List + getFieldNamesList() { + return java.util.Collections.unmodifiableList(fieldNames_); + } + public int getFieldNamesCount() { + return fieldNames_.size(); + } + public String getFieldNames(int index) { + return fieldNames_.get(index); + } + public Builder setFieldNames( + int index, String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureFieldNamesIsMutable(); + fieldNames_.set(index, value); + onChanged(); + return this; + } + public Builder addFieldNames(String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureFieldNamesIsMutable(); + fieldNames_.add(value); + onChanged(); + return this; + } + public Builder addAllFieldNames( + java.lang.Iterable values) { + ensureFieldNamesIsMutable(); + super.addAll(values, fieldNames_); + onChanged(); + return this; + } + public Builder clearFieldNames() { + fieldNames_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + void addFieldNames(com.google.protobuf.ByteString value) { + ensureFieldNamesIsMutable(); + fieldNames_.add(value); + onChanged(); + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.Type) + } + + static { + defaultInstance = new Type(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.Type) + } + + public interface StripeInformationOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional uint64 offset = 1; + boolean hasOffset(); + long getOffset(); + + // optional uint64 indexLength = 2; + boolean hasIndexLength(); + long getIndexLength(); + + // optional uint64 dataLength = 3; + boolean hasDataLength(); + long getDataLength(); + + // optional uint64 footerLength = 4; + boolean hasFooterLength(); + long getFooterLength(); + + // optional uint64 numberOfRows = 5; + boolean hasNumberOfRows(); + long getNumberOfRows(); + } + public static final class StripeInformation extends + com.google.protobuf.GeneratedMessage + implements StripeInformationOrBuilder { + // Use StripeInformation.newBuilder() to construct. + private StripeInformation(Builder builder) { + super(builder); + } + private StripeInformation(boolean noInit) {} + + private static final StripeInformation defaultInstance; + public static StripeInformation getDefaultInstance() { + return defaultInstance; + } + + public StripeInformation getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_fieldAccessorTable; + } + + private int bitField0_; + // optional uint64 offset = 1; + public static final int OFFSET_FIELD_NUMBER = 1; + private long offset_; + public boolean hasOffset() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getOffset() { + return offset_; + } + + // optional uint64 indexLength = 2; + public static final int INDEXLENGTH_FIELD_NUMBER = 2; + private long indexLength_; + public boolean hasIndexLength() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getIndexLength() { + return indexLength_; + } + + // optional uint64 dataLength = 3; + public static final int DATALENGTH_FIELD_NUMBER = 3; + private long dataLength_; + public boolean hasDataLength() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getDataLength() { + return dataLength_; + } + + // optional uint64 footerLength = 4; + public static final int FOOTERLENGTH_FIELD_NUMBER = 4; + private long footerLength_; + public boolean hasFooterLength() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public long getFooterLength() { + return footerLength_; + } + + // optional uint64 numberOfRows = 5; + public static final int NUMBEROFROWS_FIELD_NUMBER = 5; + private long numberOfRows_; + public boolean hasNumberOfRows() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public long getNumberOfRows() { + return numberOfRows_; + } + + private void initFields() { + offset_ = 0L; + indexLength_ = 0L; + dataLength_ = 0L; + footerLength_ = 0L; + numberOfRows_ = 0L; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeUInt64(1, offset_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeUInt64(2, indexLength_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeUInt64(3, dataLength_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeUInt64(4, footerLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeUInt64(5, numberOfRows_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(1, offset_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(2, indexLength_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(3, dataLength_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(4, footerLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(5, numberOfRows_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformationOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + offset_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + indexLength_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + dataLength_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + footerLength_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + numberOfRows_ = 0L; + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.offset_ = offset_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.indexLength_ = indexLength_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.dataLength_ = dataLength_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.footerLength_ = footerLength_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.numberOfRows_ = numberOfRows_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.getDefaultInstance()) return this; + if (other.hasOffset()) { + setOffset(other.getOffset()); + } + if (other.hasIndexLength()) { + setIndexLength(other.getIndexLength()); + } + if (other.hasDataLength()) { + setDataLength(other.getDataLength()); + } + if (other.hasFooterLength()) { + setFooterLength(other.getFooterLength()); + } + if (other.hasNumberOfRows()) { + setNumberOfRows(other.getNumberOfRows()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + offset_ = input.readUInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + indexLength_ = input.readUInt64(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + dataLength_ = input.readUInt64(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + footerLength_ = input.readUInt64(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + numberOfRows_ = input.readUInt64(); + break; + } + } + } + } + + private int bitField0_; + + // optional uint64 offset = 1; + private long offset_ ; + public boolean hasOffset() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getOffset() { + return offset_; + } + public Builder setOffset(long value) { + bitField0_ |= 0x00000001; + offset_ = value; + onChanged(); + return this; + } + public Builder clearOffset() { + bitField0_ = (bitField0_ & ~0x00000001); + offset_ = 0L; + onChanged(); + return this; + } + + // optional uint64 indexLength = 2; + private long indexLength_ ; + public boolean hasIndexLength() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getIndexLength() { + return indexLength_; + } + public Builder setIndexLength(long value) { + bitField0_ |= 0x00000002; + indexLength_ = value; + onChanged(); + return this; + } + public Builder clearIndexLength() { + bitField0_ = (bitField0_ & ~0x00000002); + indexLength_ = 0L; + onChanged(); + return this; + } + + // optional uint64 dataLength = 3; + private long dataLength_ ; + public boolean hasDataLength() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getDataLength() { + return dataLength_; + } + public Builder setDataLength(long value) { + bitField0_ |= 0x00000004; + dataLength_ = value; + onChanged(); + return this; + } + public Builder clearDataLength() { + bitField0_ = (bitField0_ & ~0x00000004); + dataLength_ = 0L; + onChanged(); + return this; + } + + // optional uint64 footerLength = 4; + private long footerLength_ ; + public boolean hasFooterLength() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public long getFooterLength() { + return footerLength_; + } + public Builder setFooterLength(long value) { + bitField0_ |= 0x00000008; + footerLength_ = value; + onChanged(); + return this; + } + public Builder clearFooterLength() { + bitField0_ = (bitField0_ & ~0x00000008); + footerLength_ = 0L; + onChanged(); + return this; + } + + // optional uint64 numberOfRows = 5; + private long numberOfRows_ ; + public boolean hasNumberOfRows() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public long getNumberOfRows() { + return numberOfRows_; + } + public Builder setNumberOfRows(long value) { + bitField0_ |= 0x00000010; + numberOfRows_ = value; + onChanged(); + return this; + } + public Builder clearNumberOfRows() { + bitField0_ = (bitField0_ & ~0x00000010); + numberOfRows_ = 0L; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.StripeInformation) + } + + static { + defaultInstance = new StripeInformation(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.StripeInformation) + } + + public interface UserMetadataItemOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string name = 1; + boolean hasName(); + String getName(); + + // required bytes value = 2; + boolean hasValue(); + com.google.protobuf.ByteString getValue(); + } + public static final class UserMetadataItem extends + com.google.protobuf.GeneratedMessage + implements UserMetadataItemOrBuilder { + // Use UserMetadataItem.newBuilder() to construct. + private UserMetadataItem(Builder builder) { + super(builder); + } + private UserMetadataItem(boolean noInit) {} + + private static final UserMetadataItem defaultInstance; + public static UserMetadataItem getDefaultInstance() { + return defaultInstance; + } + + public UserMetadataItem getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_fieldAccessorTable; + } + + private int bitField0_; + // required string name = 1; + public static final int NAME_FIELD_NUMBER = 1; + private java.lang.Object name_; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + name_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required bytes value = 2; + public static final int VALUE_FIELD_NUMBER = 2; + private com.google.protobuf.ByteString value_; + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public com.google.protobuf.ByteString getValue() { + return value_; + } + + private void initFields() { + name_ = ""; + value_ = com.google.protobuf.ByteString.EMPTY; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasValue()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, value_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, value_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItemOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + value_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.value_ = value_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.getDefaultInstance()) return this; + if (other.hasName()) { + setName(other.getName()); + } + if (other.hasValue()) { + setValue(other.getValue()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + + return false; + } + if (!hasValue()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + name_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + value_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // required string name = 1; + private java.lang.Object name_ = ""; + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + name_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setName(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + void setName(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + } + + // required bytes value = 2; + private com.google.protobuf.ByteString value_ = com.google.protobuf.ByteString.EMPTY; + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public com.google.protobuf.ByteString getValue() { + return value_; + } + public Builder setValue(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + value_ = value; + onChanged(); + return this; + } + public Builder clearValue() { + bitField0_ = (bitField0_ & ~0x00000002); + value_ = getDefaultInstance().getValue(); + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.UserMetadataItem) + } + + static { + defaultInstance = new UserMetadataItem(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.UserMetadataItem) + } + + public interface FooterOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional uint64 headerLength = 1; + boolean hasHeaderLength(); + long getHeaderLength(); + + // optional uint64 contentLength = 2; + boolean hasContentLength(); + long getContentLength(); + + // repeated .org.apache.hadoop.hive.ql.io.orc.StripeInformation stripes = 3; + java.util.List + getStripesList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation getStripes(int index); + int getStripesCount(); + java.util.List + getStripesOrBuilderList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformationOrBuilder getStripesOrBuilder( + int index); + + // repeated .org.apache.hadoop.hive.ql.io.orc.Type types = 4; + java.util.List + getTypesList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type getTypes(int index); + int getTypesCount(); + java.util.List + getTypesOrBuilderList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.TypeOrBuilder getTypesOrBuilder( + int index); + + // repeated .org.apache.hadoop.hive.ql.io.orc.UserMetadataItem metadata = 5; + java.util.List + getMetadataList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem getMetadata(int index); + int getMetadataCount(); + java.util.List + getMetadataOrBuilderList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItemOrBuilder getMetadataOrBuilder( + int index); + + // optional uint64 numberOfRows = 6; + boolean hasNumberOfRows(); + long getNumberOfRows(); + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics statistics = 7; + java.util.List + getStatisticsList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getStatistics(int index); + int getStatisticsCount(); + java.util.List + getStatisticsOrBuilderList(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getStatisticsOrBuilder( + int index); + + // optional uint32 rowIndexStride = 8; + boolean hasRowIndexStride(); + int getRowIndexStride(); + } + public static final class Footer extends + com.google.protobuf.GeneratedMessage + implements FooterOrBuilder { + // Use Footer.newBuilder() to construct. + private Footer(Builder builder) { + super(builder); + } + private Footer(boolean noInit) {} + + private static final Footer defaultInstance; + public static Footer getDefaultInstance() { + return defaultInstance; + } + + public Footer getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_fieldAccessorTable; + } + + private int bitField0_; + // optional uint64 headerLength = 1; + public static final int HEADERLENGTH_FIELD_NUMBER = 1; + private long headerLength_; + public boolean hasHeaderLength() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getHeaderLength() { + return headerLength_; + } + + // optional uint64 contentLength = 2; + public static final int CONTENTLENGTH_FIELD_NUMBER = 2; + private long contentLength_; + public boolean hasContentLength() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getContentLength() { + return contentLength_; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.StripeInformation stripes = 3; + public static final int STRIPES_FIELD_NUMBER = 3; + private java.util.List stripes_; + public java.util.List getStripesList() { + return stripes_; + } + public java.util.List + getStripesOrBuilderList() { + return stripes_; + } + public int getStripesCount() { + return stripes_.size(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation getStripes(int index) { + return stripes_.get(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformationOrBuilder getStripesOrBuilder( + int index) { + return stripes_.get(index); + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.Type types = 4; + public static final int TYPES_FIELD_NUMBER = 4; + private java.util.List types_; + public java.util.List getTypesList() { + return types_; + } + public java.util.List + getTypesOrBuilderList() { + return types_; + } + public int getTypesCount() { + return types_.size(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type getTypes(int index) { + return types_.get(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.TypeOrBuilder getTypesOrBuilder( + int index) { + return types_.get(index); + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.UserMetadataItem metadata = 5; + public static final int METADATA_FIELD_NUMBER = 5; + private java.util.List metadata_; + public java.util.List getMetadataList() { + return metadata_; + } + public java.util.List + getMetadataOrBuilderList() { + return metadata_; + } + public int getMetadataCount() { + return metadata_.size(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem getMetadata(int index) { + return metadata_.get(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItemOrBuilder getMetadataOrBuilder( + int index) { + return metadata_.get(index); + } + + // optional uint64 numberOfRows = 6; + public static final int NUMBEROFROWS_FIELD_NUMBER = 6; + private long numberOfRows_; + public boolean hasNumberOfRows() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getNumberOfRows() { + return numberOfRows_; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics statistics = 7; + public static final int STATISTICS_FIELD_NUMBER = 7; + private java.util.List statistics_; + public java.util.List getStatisticsList() { + return statistics_; + } + public java.util.List + getStatisticsOrBuilderList() { + return statistics_; + } + public int getStatisticsCount() { + return statistics_.size(); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getStatistics(int index) { + return statistics_.get(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getStatisticsOrBuilder( + int index) { + return statistics_.get(index); + } + + // optional uint32 rowIndexStride = 8; + public static final int ROWINDEXSTRIDE_FIELD_NUMBER = 8; + private int rowIndexStride_; + public boolean hasRowIndexStride() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getRowIndexStride() { + return rowIndexStride_; + } + + private void initFields() { + headerLength_ = 0L; + contentLength_ = 0L; + stripes_ = java.util.Collections.emptyList(); + types_ = java.util.Collections.emptyList(); + metadata_ = java.util.Collections.emptyList(); + numberOfRows_ = 0L; + statistics_ = java.util.Collections.emptyList(); + rowIndexStride_ = 0; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + for (int i = 0; i < getTypesCount(); i++) { + if (!getTypes(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + for (int i = 0; i < getMetadataCount(); i++) { + if (!getMetadata(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeUInt64(1, headerLength_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeUInt64(2, contentLength_); + } + for (int i = 0; i < stripes_.size(); i++) { + output.writeMessage(3, stripes_.get(i)); + } + for (int i = 0; i < types_.size(); i++) { + output.writeMessage(4, types_.get(i)); + } + for (int i = 0; i < metadata_.size(); i++) { + output.writeMessage(5, metadata_.get(i)); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeUInt64(6, numberOfRows_); + } + for (int i = 0; i < statistics_.size(); i++) { + output.writeMessage(7, statistics_.get(i)); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeUInt32(8, rowIndexStride_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(1, headerLength_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(2, contentLength_); + } + for (int i = 0; i < stripes_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(3, stripes_.get(i)); + } + for (int i = 0; i < types_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(4, types_.get(i)); + } + for (int i = 0; i < metadata_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(5, metadata_.get(i)); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(6, numberOfRows_); + } + for (int i = 0; i < statistics_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(7, statistics_.get(i)); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt32Size(8, rowIndexStride_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.FooterOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getStripesFieldBuilder(); + getTypesFieldBuilder(); + getMetadataFieldBuilder(); + getStatisticsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + headerLength_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + contentLength_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + if (stripesBuilder_ == null) { + stripes_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + } else { + stripesBuilder_.clear(); + } + if (typesBuilder_ == null) { + types_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000008); + } else { + typesBuilder_.clear(); + } + if (metadataBuilder_ == null) { + metadata_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000010); + } else { + metadataBuilder_.clear(); + } + numberOfRows_ = 0L; + bitField0_ = (bitField0_ & ~0x00000020); + if (statisticsBuilder_ == null) { + statistics_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000040); + } else { + statisticsBuilder_.clear(); + } + rowIndexStride_ = 0; + bitField0_ = (bitField0_ & ~0x00000080); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.headerLength_ = headerLength_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.contentLength_ = contentLength_; + if (stripesBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004)) { + stripes_ = java.util.Collections.unmodifiableList(stripes_); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.stripes_ = stripes_; + } else { + result.stripes_ = stripesBuilder_.build(); + } + if (typesBuilder_ == null) { + if (((bitField0_ & 0x00000008) == 0x00000008)) { + types_ = java.util.Collections.unmodifiableList(types_); + bitField0_ = (bitField0_ & ~0x00000008); + } + result.types_ = types_; + } else { + result.types_ = typesBuilder_.build(); + } + if (metadataBuilder_ == null) { + if (((bitField0_ & 0x00000010) == 0x00000010)) { + metadata_ = java.util.Collections.unmodifiableList(metadata_); + bitField0_ = (bitField0_ & ~0x00000010); + } + result.metadata_ = metadata_; + } else { + result.metadata_ = metadataBuilder_.build(); + } + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000004; + } + result.numberOfRows_ = numberOfRows_; + if (statisticsBuilder_ == null) { + if (((bitField0_ & 0x00000040) == 0x00000040)) { + statistics_ = java.util.Collections.unmodifiableList(statistics_); + bitField0_ = (bitField0_ & ~0x00000040); + } + result.statistics_ = statistics_; + } else { + result.statistics_ = statisticsBuilder_.build(); + } + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { + to_bitField0_ |= 0x00000008; + } + result.rowIndexStride_ = rowIndexStride_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer.getDefaultInstance()) return this; + if (other.hasHeaderLength()) { + setHeaderLength(other.getHeaderLength()); + } + if (other.hasContentLength()) { + setContentLength(other.getContentLength()); + } + if (stripesBuilder_ == null) { + if (!other.stripes_.isEmpty()) { + if (stripes_.isEmpty()) { + stripes_ = other.stripes_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureStripesIsMutable(); + stripes_.addAll(other.stripes_); + } + onChanged(); + } + } else { + if (!other.stripes_.isEmpty()) { + if (stripesBuilder_.isEmpty()) { + stripesBuilder_.dispose(); + stripesBuilder_ = null; + stripes_ = other.stripes_; + bitField0_ = (bitField0_ & ~0x00000004); + stripesBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getStripesFieldBuilder() : null; + } else { + stripesBuilder_.addAllMessages(other.stripes_); + } + } + } + if (typesBuilder_ == null) { + if (!other.types_.isEmpty()) { + if (types_.isEmpty()) { + types_ = other.types_; + bitField0_ = (bitField0_ & ~0x00000008); + } else { + ensureTypesIsMutable(); + types_.addAll(other.types_); + } + onChanged(); + } + } else { + if (!other.types_.isEmpty()) { + if (typesBuilder_.isEmpty()) { + typesBuilder_.dispose(); + typesBuilder_ = null; + types_ = other.types_; + bitField0_ = (bitField0_ & ~0x00000008); + typesBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getTypesFieldBuilder() : null; + } else { + typesBuilder_.addAllMessages(other.types_); + } + } + } + if (metadataBuilder_ == null) { + if (!other.metadata_.isEmpty()) { + if (metadata_.isEmpty()) { + metadata_ = other.metadata_; + bitField0_ = (bitField0_ & ~0x00000010); + } else { + ensureMetadataIsMutable(); + metadata_.addAll(other.metadata_); + } + onChanged(); + } + } else { + if (!other.metadata_.isEmpty()) { + if (metadataBuilder_.isEmpty()) { + metadataBuilder_.dispose(); + metadataBuilder_ = null; + metadata_ = other.metadata_; + bitField0_ = (bitField0_ & ~0x00000010); + metadataBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getMetadataFieldBuilder() : null; + } else { + metadataBuilder_.addAllMessages(other.metadata_); + } + } + } + if (other.hasNumberOfRows()) { + setNumberOfRows(other.getNumberOfRows()); + } + if (statisticsBuilder_ == null) { + if (!other.statistics_.isEmpty()) { + if (statistics_.isEmpty()) { + statistics_ = other.statistics_; + bitField0_ = (bitField0_ & ~0x00000040); + } else { + ensureStatisticsIsMutable(); + statistics_.addAll(other.statistics_); + } + onChanged(); + } + } else { + if (!other.statistics_.isEmpty()) { + if (statisticsBuilder_.isEmpty()) { + statisticsBuilder_.dispose(); + statisticsBuilder_ = null; + statistics_ = other.statistics_; + bitField0_ = (bitField0_ & ~0x00000040); + statisticsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getStatisticsFieldBuilder() : null; + } else { + statisticsBuilder_.addAllMessages(other.statistics_); + } + } + } + if (other.hasRowIndexStride()) { + setRowIndexStride(other.getRowIndexStride()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + for (int i = 0; i < getTypesCount(); i++) { + if (!getTypes(i).isInitialized()) { + + return false; + } + } + for (int i = 0; i < getMetadataCount(); i++) { + if (!getMetadata(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + headerLength_ = input.readUInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + contentLength_ = input.readUInt64(); + break; + } + case 26: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addStripes(subBuilder.buildPartial()); + break; + } + case 34: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addTypes(subBuilder.buildPartial()); + break; + } + case 42: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addMetadata(subBuilder.buildPartial()); + break; + } + case 48: { + bitField0_ |= 0x00000020; + numberOfRows_ = input.readUInt64(); + break; + } + case 58: { + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder subBuilder = org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addStatistics(subBuilder.buildPartial()); + break; + } + case 64: { + bitField0_ |= 0x00000080; + rowIndexStride_ = input.readUInt32(); + break; + } + } + } + } + + private int bitField0_; + + // optional uint64 headerLength = 1; + private long headerLength_ ; + public boolean hasHeaderLength() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getHeaderLength() { + return headerLength_; + } + public Builder setHeaderLength(long value) { + bitField0_ |= 0x00000001; + headerLength_ = value; + onChanged(); + return this; + } + public Builder clearHeaderLength() { + bitField0_ = (bitField0_ & ~0x00000001); + headerLength_ = 0L; + onChanged(); + return this; + } + + // optional uint64 contentLength = 2; + private long contentLength_ ; + public boolean hasContentLength() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getContentLength() { + return contentLength_; + } + public Builder setContentLength(long value) { + bitField0_ |= 0x00000002; + contentLength_ = value; + onChanged(); + return this; + } + public Builder clearContentLength() { + bitField0_ = (bitField0_ & ~0x00000002); + contentLength_ = 0L; + onChanged(); + return this; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.StripeInformation stripes = 3; + private java.util.List stripes_ = + java.util.Collections.emptyList(); + private void ensureStripesIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + stripes_ = new java.util.ArrayList(stripes_); + bitField0_ |= 0x00000004; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformationOrBuilder> stripesBuilder_; + + public java.util.List getStripesList() { + if (stripesBuilder_ == null) { + return java.util.Collections.unmodifiableList(stripes_); + } else { + return stripesBuilder_.getMessageList(); + } + } + public int getStripesCount() { + if (stripesBuilder_ == null) { + return stripes_.size(); + } else { + return stripesBuilder_.getCount(); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation getStripes(int index) { + if (stripesBuilder_ == null) { + return stripes_.get(index); + } else { + return stripesBuilder_.getMessage(index); + } + } + public Builder setStripes( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation value) { + if (stripesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStripesIsMutable(); + stripes_.set(index, value); + onChanged(); + } else { + stripesBuilder_.setMessage(index, value); + } + return this; + } + public Builder setStripes( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder builderForValue) { + if (stripesBuilder_ == null) { + ensureStripesIsMutable(); + stripes_.set(index, builderForValue.build()); + onChanged(); + } else { + stripesBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addStripes(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation value) { + if (stripesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStripesIsMutable(); + stripes_.add(value); + onChanged(); + } else { + stripesBuilder_.addMessage(value); + } + return this; + } + public Builder addStripes( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation value) { + if (stripesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStripesIsMutable(); + stripes_.add(index, value); + onChanged(); + } else { + stripesBuilder_.addMessage(index, value); + } + return this; + } + public Builder addStripes( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder builderForValue) { + if (stripesBuilder_ == null) { + ensureStripesIsMutable(); + stripes_.add(builderForValue.build()); + onChanged(); + } else { + stripesBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addStripes( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder builderForValue) { + if (stripesBuilder_ == null) { + ensureStripesIsMutable(); + stripes_.add(index, builderForValue.build()); + onChanged(); + } else { + stripesBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllStripes( + java.lang.Iterable values) { + if (stripesBuilder_ == null) { + ensureStripesIsMutable(); + super.addAll(values, stripes_); + onChanged(); + } else { + stripesBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearStripes() { + if (stripesBuilder_ == null) { + stripes_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + } else { + stripesBuilder_.clear(); + } + return this; + } + public Builder removeStripes(int index) { + if (stripesBuilder_ == null) { + ensureStripesIsMutable(); + stripes_.remove(index); + onChanged(); + } else { + stripesBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder getStripesBuilder( + int index) { + return getStripesFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformationOrBuilder getStripesOrBuilder( + int index) { + if (stripesBuilder_ == null) { + return stripes_.get(index); } else { + return stripesBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getStripesOrBuilderList() { + if (stripesBuilder_ != null) { + return stripesBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(stripes_); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder addStripesBuilder() { + return getStripesFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.getDefaultInstance()); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder addStripesBuilder( + int index) { + return getStripesFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.getDefaultInstance()); + } + public java.util.List + getStripesBuilderList() { + return getStripesFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformationOrBuilder> + getStripesFieldBuilder() { + if (stripesBuilder_ == null) { + stripesBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformationOrBuilder>( + stripes_, + ((bitField0_ & 0x00000004) == 0x00000004), + getParentForChildren(), + isClean()); + stripes_ = null; + } + return stripesBuilder_; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.Type types = 4; + private java.util.List types_ = + java.util.Collections.emptyList(); + private void ensureTypesIsMutable() { + if (!((bitField0_ & 0x00000008) == 0x00000008)) { + types_ = new java.util.ArrayList(types_); + bitField0_ |= 0x00000008; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type, org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.TypeOrBuilder> typesBuilder_; + + public java.util.List getTypesList() { + if (typesBuilder_ == null) { + return java.util.Collections.unmodifiableList(types_); + } else { + return typesBuilder_.getMessageList(); + } + } + public int getTypesCount() { + if (typesBuilder_ == null) { + return types_.size(); + } else { + return typesBuilder_.getCount(); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type getTypes(int index) { + if (typesBuilder_ == null) { + return types_.get(index); + } else { + return typesBuilder_.getMessage(index); + } + } + public Builder setTypes( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Type value) { + if (typesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureTypesIsMutable(); + types_.set(index, value); + onChanged(); + } else { + typesBuilder_.setMessage(index, value); + } + return this; + } + public Builder setTypes( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder builderForValue) { + if (typesBuilder_ == null) { + ensureTypesIsMutable(); + types_.set(index, builderForValue.build()); + onChanged(); + } else { + typesBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addTypes(org.apache.hadoop.hive.ql.io.orc.OrcProto.Type value) { + if (typesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureTypesIsMutable(); + types_.add(value); + onChanged(); + } else { + typesBuilder_.addMessage(value); + } + return this; + } + public Builder addTypes( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Type value) { + if (typesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureTypesIsMutable(); + types_.add(index, value); + onChanged(); + } else { + typesBuilder_.addMessage(index, value); + } + return this; + } + public Builder addTypes( + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder builderForValue) { + if (typesBuilder_ == null) { + ensureTypesIsMutable(); + types_.add(builderForValue.build()); + onChanged(); + } else { + typesBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addTypes( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder builderForValue) { + if (typesBuilder_ == null) { + ensureTypesIsMutable(); + types_.add(index, builderForValue.build()); + onChanged(); + } else { + typesBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllTypes( + java.lang.Iterable values) { + if (typesBuilder_ == null) { + ensureTypesIsMutable(); + super.addAll(values, types_); + onChanged(); + } else { + typesBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearTypes() { + if (typesBuilder_ == null) { + types_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000008); + onChanged(); + } else { + typesBuilder_.clear(); + } + return this; + } + public Builder removeTypes(int index) { + if (typesBuilder_ == null) { + ensureTypesIsMutable(); + types_.remove(index); + onChanged(); + } else { + typesBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder getTypesBuilder( + int index) { + return getTypesFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.TypeOrBuilder getTypesOrBuilder( + int index) { + if (typesBuilder_ == null) { + return types_.get(index); } else { + return typesBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getTypesOrBuilderList() { + if (typesBuilder_ != null) { + return typesBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(types_); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder addTypesBuilder() { + return getTypesFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.getDefaultInstance()); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder addTypesBuilder( + int index) { + return getTypesFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.getDefaultInstance()); + } + public java.util.List + getTypesBuilderList() { + return getTypesFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type, org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.TypeOrBuilder> + getTypesFieldBuilder() { + if (typesBuilder_ == null) { + typesBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type, org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.TypeOrBuilder>( + types_, + ((bitField0_ & 0x00000008) == 0x00000008), + getParentForChildren(), + isClean()); + types_ = null; + } + return typesBuilder_; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.UserMetadataItem metadata = 5; + private java.util.List metadata_ = + java.util.Collections.emptyList(); + private void ensureMetadataIsMutable() { + if (!((bitField0_ & 0x00000010) == 0x00000010)) { + metadata_ = new java.util.ArrayList(metadata_); + bitField0_ |= 0x00000010; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItemOrBuilder> metadataBuilder_; + + public java.util.List getMetadataList() { + if (metadataBuilder_ == null) { + return java.util.Collections.unmodifiableList(metadata_); + } else { + return metadataBuilder_.getMessageList(); + } + } + public int getMetadataCount() { + if (metadataBuilder_ == null) { + return metadata_.size(); + } else { + return metadataBuilder_.getCount(); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem getMetadata(int index) { + if (metadataBuilder_ == null) { + return metadata_.get(index); + } else { + return metadataBuilder_.getMessage(index); + } + } + public Builder setMetadata( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem value) { + if (metadataBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureMetadataIsMutable(); + metadata_.set(index, value); + onChanged(); + } else { + metadataBuilder_.setMessage(index, value); + } + return this; + } + public Builder setMetadata( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder builderForValue) { + if (metadataBuilder_ == null) { + ensureMetadataIsMutable(); + metadata_.set(index, builderForValue.build()); + onChanged(); + } else { + metadataBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addMetadata(org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem value) { + if (metadataBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureMetadataIsMutable(); + metadata_.add(value); + onChanged(); + } else { + metadataBuilder_.addMessage(value); + } + return this; + } + public Builder addMetadata( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem value) { + if (metadataBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureMetadataIsMutable(); + metadata_.add(index, value); + onChanged(); + } else { + metadataBuilder_.addMessage(index, value); + } + return this; + } + public Builder addMetadata( + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder builderForValue) { + if (metadataBuilder_ == null) { + ensureMetadataIsMutable(); + metadata_.add(builderForValue.build()); + onChanged(); + } else { + metadataBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addMetadata( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder builderForValue) { + if (metadataBuilder_ == null) { + ensureMetadataIsMutable(); + metadata_.add(index, builderForValue.build()); + onChanged(); + } else { + metadataBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllMetadata( + java.lang.Iterable values) { + if (metadataBuilder_ == null) { + ensureMetadataIsMutable(); + super.addAll(values, metadata_); + onChanged(); + } else { + metadataBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearMetadata() { + if (metadataBuilder_ == null) { + metadata_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000010); + onChanged(); + } else { + metadataBuilder_.clear(); + } + return this; + } + public Builder removeMetadata(int index) { + if (metadataBuilder_ == null) { + ensureMetadataIsMutable(); + metadata_.remove(index); + onChanged(); + } else { + metadataBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder getMetadataBuilder( + int index) { + return getMetadataFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItemOrBuilder getMetadataOrBuilder( + int index) { + if (metadataBuilder_ == null) { + return metadata_.get(index); } else { + return metadataBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getMetadataOrBuilderList() { + if (metadataBuilder_ != null) { + return metadataBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(metadata_); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder addMetadataBuilder() { + return getMetadataFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.getDefaultInstance()); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder addMetadataBuilder( + int index) { + return getMetadataFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.getDefaultInstance()); + } + public java.util.List + getMetadataBuilderList() { + return getMetadataFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItemOrBuilder> + getMetadataFieldBuilder() { + if (metadataBuilder_ == null) { + metadataBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItemOrBuilder>( + metadata_, + ((bitField0_ & 0x00000010) == 0x00000010), + getParentForChildren(), + isClean()); + metadata_ = null; + } + return metadataBuilder_; + } + + // optional uint64 numberOfRows = 6; + private long numberOfRows_ ; + public boolean hasNumberOfRows() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public long getNumberOfRows() { + return numberOfRows_; + } + public Builder setNumberOfRows(long value) { + bitField0_ |= 0x00000020; + numberOfRows_ = value; + onChanged(); + return this; + } + public Builder clearNumberOfRows() { + bitField0_ = (bitField0_ & ~0x00000020); + numberOfRows_ = 0L; + onChanged(); + return this; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics statistics = 7; + private java.util.List statistics_ = + java.util.Collections.emptyList(); + private void ensureStatisticsIsMutable() { + if (!((bitField0_ & 0x00000040) == 0x00000040)) { + statistics_ = new java.util.ArrayList(statistics_); + bitField0_ |= 0x00000040; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder> statisticsBuilder_; + + public java.util.List getStatisticsList() { + if (statisticsBuilder_ == null) { + return java.util.Collections.unmodifiableList(statistics_); + } else { + return statisticsBuilder_.getMessageList(); + } + } + public int getStatisticsCount() { + if (statisticsBuilder_ == null) { + return statistics_.size(); + } else { + return statisticsBuilder_.getCount(); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getStatistics(int index) { + if (statisticsBuilder_ == null) { + return statistics_.get(index); + } else { + return statisticsBuilder_.getMessage(index); + } + } + public Builder setStatistics( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (statisticsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStatisticsIsMutable(); + statistics_.set(index, value); + onChanged(); + } else { + statisticsBuilder_.setMessage(index, value); + } + return this; + } + public Builder setStatistics( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (statisticsBuilder_ == null) { + ensureStatisticsIsMutable(); + statistics_.set(index, builderForValue.build()); + onChanged(); + } else { + statisticsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + public Builder addStatistics(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (statisticsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStatisticsIsMutable(); + statistics_.add(value); + onChanged(); + } else { + statisticsBuilder_.addMessage(value); + } + return this; + } + public Builder addStatistics( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (statisticsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStatisticsIsMutable(); + statistics_.add(index, value); + onChanged(); + } else { + statisticsBuilder_.addMessage(index, value); + } + return this; + } + public Builder addStatistics( + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (statisticsBuilder_ == null) { + ensureStatisticsIsMutable(); + statistics_.add(builderForValue.build()); + onChanged(); + } else { + statisticsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + public Builder addStatistics( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (statisticsBuilder_ == null) { + ensureStatisticsIsMutable(); + statistics_.add(index, builderForValue.build()); + onChanged(); + } else { + statisticsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + public Builder addAllStatistics( + java.lang.Iterable values) { + if (statisticsBuilder_ == null) { + ensureStatisticsIsMutable(); + super.addAll(values, statistics_); + onChanged(); + } else { + statisticsBuilder_.addAllMessages(values); + } + return this; + } + public Builder clearStatistics() { + if (statisticsBuilder_ == null) { + statistics_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000040); + onChanged(); + } else { + statisticsBuilder_.clear(); + } + return this; + } + public Builder removeStatistics(int index) { + if (statisticsBuilder_ == null) { + ensureStatisticsIsMutable(); + statistics_.remove(index); + onChanged(); + } else { + statisticsBuilder_.remove(index); + } + return this; + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder getStatisticsBuilder( + int index) { + return getStatisticsFieldBuilder().getBuilder(index); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getStatisticsOrBuilder( + int index) { + if (statisticsBuilder_ == null) { + return statistics_.get(index); } else { + return statisticsBuilder_.getMessageOrBuilder(index); + } + } + public java.util.List + getStatisticsOrBuilderList() { + if (statisticsBuilder_ != null) { + return statisticsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(statistics_); + } + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder addStatisticsBuilder() { + return getStatisticsFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance()); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder addStatisticsBuilder( + int index) { + return getStatisticsFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance()); + } + public java.util.List + getStatisticsBuilderList() { + return getStatisticsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder> + getStatisticsFieldBuilder() { + if (statisticsBuilder_ == null) { + statisticsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder>( + statistics_, + ((bitField0_ & 0x00000040) == 0x00000040), + getParentForChildren(), + isClean()); + statistics_ = null; + } + return statisticsBuilder_; + } + + // optional uint32 rowIndexStride = 8; + private int rowIndexStride_ ; + public boolean hasRowIndexStride() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + public int getRowIndexStride() { + return rowIndexStride_; + } + public Builder setRowIndexStride(int value) { + bitField0_ |= 0x00000080; + rowIndexStride_ = value; + onChanged(); + return this; + } + public Builder clearRowIndexStride() { + bitField0_ = (bitField0_ & ~0x00000080); + rowIndexStride_ = 0; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.Footer) + } + + static { + defaultInstance = new Footer(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.Footer) + } + + public interface PostScriptOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional uint64 footerLength = 1; + boolean hasFooterLength(); + long getFooterLength(); + + // optional .org.apache.hadoop.hive.ql.io.orc.CompressionKind compression = 2; + boolean hasCompression(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind getCompression(); + + // optional uint64 compressionBlockSize = 3; + boolean hasCompressionBlockSize(); + long getCompressionBlockSize(); + } + public static final class PostScript extends + com.google.protobuf.GeneratedMessage + implements PostScriptOrBuilder { + // Use PostScript.newBuilder() to construct. + private PostScript(Builder builder) { + super(builder); + } + private PostScript(boolean noInit) {} + + private static final PostScript defaultInstance; + public static PostScript getDefaultInstance() { + return defaultInstance; + } + + public PostScript getDefaultInstanceForType() { + return defaultInstance; + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_fieldAccessorTable; + } + + private int bitField0_; + // optional uint64 footerLength = 1; + public static final int FOOTERLENGTH_FIELD_NUMBER = 1; + private long footerLength_; + public boolean hasFooterLength() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getFooterLength() { + return footerLength_; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.CompressionKind compression = 2; + public static final int COMPRESSION_FIELD_NUMBER = 2; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind compression_; + public boolean hasCompression() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind getCompression() { + return compression_; + } + + // optional uint64 compressionBlockSize = 3; + public static final int COMPRESSIONBLOCKSIZE_FIELD_NUMBER = 3; + private long compressionBlockSize_; + public boolean hasCompressionBlockSize() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getCompressionBlockSize() { + return compressionBlockSize_; + } + + private void initFields() { + footerLength_ = 0L; + compression_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind.NONE; + compressionBlockSize_ = 0L; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeUInt64(1, footerLength_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeEnum(2, compression_.getNumber()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeUInt64(3, compressionBlockSize_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(1, footerLength_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(2, compression_.getNumber()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(3, compressionBlockSize_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScriptOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_fieldAccessorTable; + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + footerLength_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + compression_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind.NONE; + bitField0_ = (bitField0_ & ~0x00000002); + compressionBlockSize_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript.getDescriptor(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.footerLength_ = footerLength_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.compression_ = compression_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.compressionBlockSize_ = compressionBlockSize_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript.getDefaultInstance()) return this; + if (other.hasFooterLength()) { + setFooterLength(other.getFooterLength()); + } + if (other.hasCompression()) { + setCompression(other.getCompression()); + } + if (other.hasCompressionBlockSize()) { + setCompressionBlockSize(other.getCompressionBlockSize()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder( + this.getUnknownFields()); + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + this.setUnknownFields(unknownFields.build()); + onChanged(); + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + footerLength_ = input.readUInt64(); + break; + } + case 16: { + int rawValue = input.readEnum(); + org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind value = org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(2, rawValue); + } else { + bitField0_ |= 0x00000002; + compression_ = value; + } + break; + } + case 24: { + bitField0_ |= 0x00000004; + compressionBlockSize_ = input.readUInt64(); + break; + } + } + } + } + + private int bitField0_; + + // optional uint64 footerLength = 1; + private long footerLength_ ; + public boolean hasFooterLength() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getFooterLength() { + return footerLength_; + } + public Builder setFooterLength(long value) { + bitField0_ |= 0x00000001; + footerLength_ = value; + onChanged(); + return this; + } + public Builder clearFooterLength() { + bitField0_ = (bitField0_ & ~0x00000001); + footerLength_ = 0L; + onChanged(); + return this; + } + + // optional .org.apache.hadoop.hive.ql.io.orc.CompressionKind compression = 2; + private org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind compression_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind.NONE; + public boolean hasCompression() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind getCompression() { + return compression_; + } + public Builder setCompression(org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + compression_ = value; + onChanged(); + return this; + } + public Builder clearCompression() { + bitField0_ = (bitField0_ & ~0x00000002); + compression_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind.NONE; + onChanged(); + return this; + } + + // optional uint64 compressionBlockSize = 3; + private long compressionBlockSize_ ; + public boolean hasCompressionBlockSize() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public long getCompressionBlockSize() { + return compressionBlockSize_; + } + public Builder setCompressionBlockSize(long value) { + bitField0_ |= 0x00000004; + compressionBlockSize_ = value; + onChanged(); + return this; + } + public Builder clearCompressionBlockSize() { + bitField0_ = (bitField0_ & ~0x00000004); + compressionBlockSize_ = 0L; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.PostScript) + } + + static { + defaultInstance = new PostScript(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.PostScript) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_IntegerStatistics_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_IntegerStatistics_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_DoubleStatistics_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_DoubleStatistics_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_StringStatistics_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_StringStatistics_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_BucketStatistics_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_BucketStatistics_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_Type_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_Type_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\032contrib/if/orc_proto.proto\022 org.apache" + + ".hadoop.hive.ql.io.orc\"B\n\021IntegerStatist" + + "ics\022\017\n\007minimum\030\001 \001(\022\022\017\n\007maximum\030\002 \001(\022\022\013\n" + + "\003sum\030\003 \001(\022\"A\n\020DoubleStatistics\022\017\n\007minimu" + + "m\030\001 \001(\001\022\017\n\007maximum\030\002 \001(\001\022\013\n\003sum\030\003 \001(\001\"4\n" + + "\020StringStatistics\022\017\n\007minimum\030\001 \001(\t\022\017\n\007ma" + + "ximum\030\002 \001(\t\"%\n\020BucketStatistics\022\021\n\005count" + + "\030\001 \003(\004B\002\020\001\"\340\002\n\020ColumnStatistics\022\026\n\016numbe" + + "rOfValues\030\001 \001(\004\022J\n\rintStatistics\030\002 \001(\01323" + + ".org.apache.hadoop.hive.ql.io.orc.Intege", + "rStatistics\022L\n\020doubleStatistics\030\003 \001(\01322." + + "org.apache.hadoop.hive.ql.io.orc.DoubleS" + + "tatistics\022L\n\020stringStatistics\030\004 \001(\01322.or" + + "g.apache.hadoop.hive.ql.io.orc.StringSta" + + "tistics\022L\n\020bucketStatistics\030\005 \001(\01322.org." + + "apache.hadoop.hive.ql.io.orc.BucketStati" + + "stics\"n\n\rRowIndexEntry\022\025\n\tpositions\030\001 \003(" + + "\004B\002\020\001\022F\n\nstatistics\030\002 \001(\01322.org.apache.h" + + "adoop.hive.ql.io.orc.ColumnStatistics\"J\n" + + "\010RowIndex\022>\n\005entry\030\001 \003(\0132/.org.apache.ha", + "doop.hive.ql.io.orc.RowIndexEntry\"\331\001\n\006St" + + "ream\022;\n\004kind\030\001 \002(\0162-.org.apache.hadoop.h" + + "ive.ql.io.orc.Stream.Kind\022\016\n\006column\030\002 \001(" + + "\r\022\016\n\006length\030\003 \001(\004\"r\n\004Kind\022\013\n\007PRESENT\020\000\022\010" + + "\n\004DATA\020\001\022\n\n\006LENGTH\020\002\022\023\n\017DICTIONARY_DATA\020" + + "\003\022\024\n\020DICTIONARY_COUNT\020\004\022\r\n\tNANO_DATA\020\005\022\r" + + "\n\tROW_INDEX\020\006\"\220\001\n\014StripeFooter\0229\n\007stream" + + "s\030\001 \003(\0132(.org.apache.hadoop.hive.ql.io.o" + + "rc.Stream\022E\n\007columns\030\002 \003(\01620.org.apache." + + "hadoop.hive.ql.io.orc.ColumnEncodingB\002\020\001", + "\"\221\002\n\004Type\0229\n\004kind\030\001 \002(\0162+.org.apache.had" + + "oop.hive.ql.io.orc.Type.Kind\022\024\n\010subtypes" + + "\030\002 \003(\rB\002\020\001\022\022\n\nfieldNames\030\003 \003(\t\"\243\001\n\004Kind\022" + + "\013\n\007BOOLEAN\020\000\022\010\n\004BYTE\020\001\022\t\n\005SHORT\020\002\022\007\n\003INT" + + "\020\003\022\010\n\004LONG\020\004\022\t\n\005FLOAT\020\005\022\n\n\006DOUBLE\020\006\022\n\n\006S" + + "TRING\020\007\022\n\n\006BINARY\020\010\022\r\n\tTIMESTAMP\020\t\022\010\n\004LI" + + "ST\020\n\022\007\n\003MAP\020\013\022\n\n\006STRUCT\020\014\022\t\n\005UNION\020\r\"x\n\021" + + "StripeInformation\022\016\n\006offset\030\001 \001(\004\022\023\n\013ind" + + "exLength\030\002 \001(\004\022\022\n\ndataLength\030\003 \001(\004\022\024\n\014fo" + + "oterLength\030\004 \001(\004\022\024\n\014numberOfRows\030\005 \001(\004\"/", + "\n\020UserMetadataItem\022\014\n\004name\030\001 \002(\t\022\r\n\005valu" + + "e\030\002 \002(\014\"\356\002\n\006Footer\022\024\n\014headerLength\030\001 \001(\004" + + "\022\025\n\rcontentLength\030\002 \001(\004\022D\n\007stripes\030\003 \003(\013" + + "23.org.apache.hadoop.hive.ql.io.orc.Stri" + + "peInformation\0225\n\005types\030\004 \003(\0132&.org.apach" + + "e.hadoop.hive.ql.io.orc.Type\022D\n\010metadata" + + "\030\005 \003(\01322.org.apache.hadoop.hive.ql.io.or" + + "c.UserMetadataItem\022\024\n\014numberOfRows\030\006 \001(\004" + + "\022F\n\nstatistics\030\007 \003(\01322.org.apache.hadoop" + + ".hive.ql.io.orc.ColumnStatistics\022\026\n\016rowI", + "ndexStride\030\010 \001(\r\"\210\001\n\nPostScript\022\024\n\014foote" + + "rLength\030\001 \001(\004\022F\n\013compression\030\002 \001(\01621.org" + + ".apache.hadoop.hive.ql.io.orc.Compressio" + + "nKind\022\034\n\024compressionBlockSize\030\003 \001(\004*,\n\016C" + + "olumnEncoding\022\n\n\006DIRECT\020\000\022\016\n\nDICTIONARY\020" + + "\001*:\n\017CompressionKind\022\010\n\004NONE\020\000\022\010\n\004ZLIB\020\001" + + "\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_hadoop_hive_ql_io_orc_IntegerStatistics_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_hadoop_hive_ql_io_orc_IntegerStatistics_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_IntegerStatistics_descriptor, + new java.lang.String[] { "Minimum", "Maximum", "Sum", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.IntegerStatistics.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_DoubleStatistics_descriptor = + getDescriptor().getMessageTypes().get(1); + internal_static_org_apache_hadoop_hive_ql_io_orc_DoubleStatistics_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_DoubleStatistics_descriptor, + new java.lang.String[] { "Minimum", "Maximum", "Sum", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.DoubleStatistics.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_StringStatistics_descriptor = + getDescriptor().getMessageTypes().get(2); + internal_static_org_apache_hadoop_hive_ql_io_orc_StringStatistics_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_StringStatistics_descriptor, + new java.lang.String[] { "Minimum", "Maximum", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.StringStatistics.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_BucketStatistics_descriptor = + getDescriptor().getMessageTypes().get(3); + internal_static_org_apache_hadoop_hive_ql_io_orc_BucketStatistics_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_BucketStatistics_descriptor, + new java.lang.String[] { "Count", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.BucketStatistics.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_descriptor = + getDescriptor().getMessageTypes().get(4); + internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_ColumnStatistics_descriptor, + new java.lang.String[] { "NumberOfValues", "IntStatistics", "DoubleStatistics", "StringStatistics", "BucketStatistics", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_descriptor = + getDescriptor().getMessageTypes().get(5); + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndexEntry_descriptor, + new java.lang.String[] { "Positions", "Statistics", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_descriptor = + getDescriptor().getMessageTypes().get(6); + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_RowIndex_descriptor, + new java.lang.String[] { "Entry", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_descriptor = + getDescriptor().getMessageTypes().get(7); + internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_Stream_descriptor, + new java.lang.String[] { "Kind", "Column", "Length", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.Stream.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_descriptor = + getDescriptor().getMessageTypes().get(8); + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeFooter_descriptor, + new java.lang.String[] { "Streams", "Columns", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_Type_descriptor = + getDescriptor().getMessageTypes().get(9); + internal_static_org_apache_hadoop_hive_ql_io_orc_Type_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_Type_descriptor, + new java.lang.String[] { "Kind", "Subtypes", "FieldNames", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.Type.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_descriptor = + getDescriptor().getMessageTypes().get(10); + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeInformation_descriptor, + new java.lang.String[] { "Offset", "IndexLength", "DataLength", "FooterLength", "NumberOfRows", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeInformation.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_descriptor = + getDescriptor().getMessageTypes().get(11); + internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_descriptor, + new java.lang.String[] { "Name", "Value", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor = + getDescriptor().getMessageTypes().get(12); + internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor, + new java.lang.String[] { "HeaderLength", "ContentLength", "Stripes", "Types", "Metadata", "NumberOfRows", "Statistics", "RowIndexStride", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.Footer.Builder.class); + internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor = + getDescriptor().getMessageTypes().get(13); + internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor, + new java.lang.String[] { "FooterLength", "Compression", "CompressionBlockSize", }, + org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript.class, + org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript.Builder.class); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} Index: contrib/if/orc_proto.proto =================================================================== --- contrib/if/orc_proto.proto (revision 0) +++ contrib/if/orc_proto.proto (working copy) @@ -0,0 +1,126 @@ +package org.apache.hadoop.hive.ql.io.orc; + +message IntegerStatistics { + optional sint64 minimum = 1; + optional sint64 maximum = 2; + optional sint64 sum = 3; +} + +message DoubleStatistics { + optional double minimum = 1; + optional double maximum = 2; + optional double sum = 3; +} + +message StringStatistics { + optional string minimum = 1; + optional string maximum = 2; +} + +message BucketStatistics { + repeated uint64 count = 1 [packed=true]; +} + +message ColumnStatistics { + optional uint64 numberOfValues = 1; + optional IntegerStatistics intStatistics = 2; + optional DoubleStatistics doubleStatistics = 3; + optional StringStatistics stringStatistics = 4; + optional BucketStatistics bucketStatistics = 5; +} + +message RowIndexEntry { + repeated uint64 positions = 1 [packed=true]; + optional ColumnStatistics statistics = 2; +} + +message RowIndex { + repeated RowIndexEntry entry = 1; +} + +message Stream { + // if you add new index stream kinds, you need to make sure to update + // StreamName to ensure it is added to the stripe in the right area + enum Kind { + PRESENT = 0; + DATA = 1; + LENGTH = 2; + DICTIONARY_DATA = 3; + DICTIONARY_COUNT = 4; + NANO_DATA = 5; + ROW_INDEX = 6; + } + required Kind kind = 1; + optional uint32 column = 2; + optional uint64 length = 3; +} + +enum ColumnEncoding { + DIRECT = 0; + DICTIONARY = 1; +} + +message StripeFooter { + repeated Stream streams = 1; + repeated ColumnEncoding columns = 2 [packed = true]; +} + +message Type { + enum Kind { + BOOLEAN = 0; + BYTE = 1; + SHORT = 2; + INT = 3; + LONG = 4; + FLOAT = 5; + DOUBLE = 6; + STRING = 7; + BINARY = 8; + TIMESTAMP = 9; + LIST = 10; + MAP = 11; + STRUCT = 12; + UNION = 13; + } + required Kind kind = 1; + repeated uint32 subtypes = 2 [packed=true]; + repeated string fieldNames = 3; +} + +message StripeInformation { + optional uint64 offset = 1; + optional uint64 indexLength = 2; + optional uint64 dataLength = 3; + optional uint64 footerLength = 4; + optional uint64 numberOfRows = 5; +} + +message UserMetadataItem { + required string name = 1; + required bytes value = 2; +} + +message Footer { + optional uint64 headerLength = 1; + optional uint64 contentLength = 2; + repeated StripeInformation stripes = 3; + repeated Type types = 4; + repeated UserMetadataItem metadata = 5; + optional uint64 numberOfRows = 6; + repeated ColumnStatistics statistics = 7; + optional uint32 rowIndexStride = 8; +} + +enum CompressionKind { + NONE = 0; + ZLIB = 1; + SNAPPY = 2; + LZO = 3; +} + +// Serialized length must be less that 255 bytes +message PostScript { + optional uint64 footerLength = 1; + optional CompressionKind compression = 2; + optional uint64 compressionBlockSize = 3; +} Index: contrib/build.xml =================================================================== --- contrib/build.xml (revision 1440302) +++ contrib/build.xml (working copy) @@ -28,6 +28,21 @@ + + + + + + + + Index: ivy/libraries.properties =================================================================== --- ivy/libraries.properties (revision 1440302) +++ ivy/libraries.properties (working copy) @@ -67,3 +67,5 @@ zookeeper.version=3.4.3 javolution.version=5.5.1 stringtemplate.version=3.1-b1 +protobuf-java.version=2.4.1 +snappy.version=0.2 Index: common/ivy.xml =================================================================== --- common/ivy.xml (revision 1440302) +++ common/ivy.xml (working copy) @@ -92,5 +92,11 @@ transitive="false"/> + + + + Index: ql/ivy.xml =================================================================== --- ql/ivy.xml (revision 1440302) +++ ql/ivy.xml (working copy) @@ -45,6 +45,13 @@ + +