diff --git ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java index 9409b8f..fac0cbb 100644 --- ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java +++ ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java @@ -12128,6 +12128,1354 @@ public Builder clearValue() { // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.UserMetadataItem) } + public interface StripeStatisticsOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + java.util.List + getColStatsList(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getColStats(int index); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + int getColStatsCount(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + java.util.List + getColStatsOrBuilderList(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getColStatsOrBuilder( + int index); + } + /** + * Protobuf type {@code org.apache.hadoop.hive.ql.io.orc.StripeStatistics} + */ + public static final class StripeStatistics extends + com.google.protobuf.GeneratedMessage + implements StripeStatisticsOrBuilder { + // Use StripeStatistics.newBuilder() to construct. + private StripeStatistics(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private StripeStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final StripeStatistics defaultInstance; + public static StripeStatistics getDefaultInstance() { + return defaultInstance; + } + + public StripeStatistics getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private StripeStatistics( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + colStats_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000001; + } + colStats_.add(input.readMessage(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.PARSER, extensionRegistry)); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + colStats_ = java.util.Collections.unmodifiableList(colStats_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public StripeStatistics parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new StripeStatistics(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + public static final int COLSTATS_FIELD_NUMBER = 1; + private java.util.List colStats_; + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List getColStatsList() { + return colStats_; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List + getColStatsOrBuilderList() { + return colStats_; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public int getColStatsCount() { + return colStats_.size(); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getColStats(int index) { + return colStats_.get(index); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getColStatsOrBuilder( + int index) { + return colStats_.get(index); + } + + private void initFields() { + colStats_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < colStats_.size(); i++) { + output.writeMessage(1, colStats_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < colStats_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, colStats_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.apache.hadoop.hive.ql.io.orc.StripeStatistics} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder.class); + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getColStatsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (colStatsBuilder_ == null) { + colStats_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + colStatsBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics(this); + int from_bitField0_ = bitField0_; + if (colStatsBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + colStats_ = java.util.Collections.unmodifiableList(colStats_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.colStats_ = colStats_; + } else { + result.colStats_ = colStatsBuilder_.build(); + } + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.getDefaultInstance()) return this; + if (colStatsBuilder_ == null) { + if (!other.colStats_.isEmpty()) { + if (colStats_.isEmpty()) { + colStats_ = other.colStats_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureColStatsIsMutable(); + colStats_.addAll(other.colStats_); + } + onChanged(); + } + } else { + if (!other.colStats_.isEmpty()) { + if (colStatsBuilder_.isEmpty()) { + colStatsBuilder_.dispose(); + colStatsBuilder_ = null; + colStats_ = other.colStats_; + bitField0_ = (bitField0_ & ~0x00000001); + colStatsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getColStatsFieldBuilder() : null; + } else { + colStatsBuilder_.addAllMessages(other.colStats_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + private java.util.List colStats_ = + java.util.Collections.emptyList(); + private void ensureColStatsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + colStats_ = new java.util.ArrayList(colStats_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder> colStatsBuilder_; + + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List getColStatsList() { + if (colStatsBuilder_ == null) { + return java.util.Collections.unmodifiableList(colStats_); + } else { + return colStatsBuilder_.getMessageList(); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public int getColStatsCount() { + if (colStatsBuilder_ == null) { + return colStats_.size(); + } else { + return colStatsBuilder_.getCount(); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getColStats(int index) { + if (colStatsBuilder_ == null) { + return colStats_.get(index); + } else { + return colStatsBuilder_.getMessage(index); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder setColStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (colStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureColStatsIsMutable(); + colStats_.set(index, value); + onChanged(); + } else { + colStatsBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder setColStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + colStats_.set(index, builderForValue.build()); + onChanged(); + } else { + colStatsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addColStats(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (colStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureColStatsIsMutable(); + colStats_.add(value); + onChanged(); + } else { + colStatsBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addColStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (colStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureColStatsIsMutable(); + colStats_.add(index, value); + onChanged(); + } else { + colStatsBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addColStats( + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + colStats_.add(builderForValue.build()); + onChanged(); + } else { + colStatsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addColStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + colStats_.add(index, builderForValue.build()); + onChanged(); + } else { + colStatsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addAllColStats( + java.lang.Iterable values) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + super.addAll(values, colStats_); + onChanged(); + } else { + colStatsBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder clearColStats() { + if (colStatsBuilder_ == null) { + colStats_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + colStatsBuilder_.clear(); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder removeColStats(int index) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + colStats_.remove(index); + onChanged(); + } else { + colStatsBuilder_.remove(index); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder getColStatsBuilder( + int index) { + return getColStatsFieldBuilder().getBuilder(index); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getColStatsOrBuilder( + int index) { + if (colStatsBuilder_ == null) { + return colStats_.get(index); } else { + return colStatsBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List + getColStatsOrBuilderList() { + if (colStatsBuilder_ != null) { + return colStatsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(colStats_); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder addColStatsBuilder() { + return getColStatsFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance()); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder addColStatsBuilder( + int index) { + return getColStatsFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance()); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List + getColStatsBuilderList() { + return getColStatsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder> + getColStatsFieldBuilder() { + if (colStatsBuilder_ == null) { + colStatsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder>( + colStats_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + colStats_ = null; + } + return colStatsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.StripeStatistics) + } + + static { + defaultInstance = new StripeStatistics(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.StripeStatistics) + } + + public interface MetadataOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + java.util.List + getStripeStatsList(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics getStripeStats(int index); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + int getStripeStatsCount(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + java.util.List + getStripeStatsOrBuilderList(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder getStripeStatsOrBuilder( + int index); + } + /** + * Protobuf type {@code org.apache.hadoop.hive.ql.io.orc.Metadata} + */ + public static final class Metadata extends + com.google.protobuf.GeneratedMessage + implements MetadataOrBuilder { + // Use Metadata.newBuilder() to construct. + private Metadata(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private Metadata(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final Metadata defaultInstance; + public static Metadata getDefaultInstance() { + return defaultInstance; + } + + public Metadata getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private Metadata( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + stripeStats_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000001; + } + stripeStats_.add(input.readMessage(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.PARSER, extensionRegistry)); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + stripeStats_ = java.util.Collections.unmodifiableList(stripeStats_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public Metadata parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new Metadata(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + public static final int STRIPESTATS_FIELD_NUMBER = 1; + private java.util.List stripeStats_; + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List getStripeStatsList() { + return stripeStats_; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List + getStripeStatsOrBuilderList() { + return stripeStats_; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public int getStripeStatsCount() { + return stripeStats_.size(); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics getStripeStats(int index) { + return stripeStats_.get(index); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder getStripeStatsOrBuilder( + int index) { + return stripeStats_.get(index); + } + + private void initFields() { + stripeStats_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < stripeStats_.size(); i++) { + output.writeMessage(1, stripeStats_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < stripeStats_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, stripeStats_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.apache.hadoop.hive.ql.io.orc.Metadata} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.MetadataOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.Builder.class); + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getStripeStatsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (stripeStatsBuilder_ == null) { + stripeStats_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + stripeStatsBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata(this); + int from_bitField0_ = bitField0_; + if (stripeStatsBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + stripeStats_ = java.util.Collections.unmodifiableList(stripeStats_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.stripeStats_ = stripeStats_; + } else { + result.stripeStats_ = stripeStatsBuilder_.build(); + } + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.getDefaultInstance()) return this; + if (stripeStatsBuilder_ == null) { + if (!other.stripeStats_.isEmpty()) { + if (stripeStats_.isEmpty()) { + stripeStats_ = other.stripeStats_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureStripeStatsIsMutable(); + stripeStats_.addAll(other.stripeStats_); + } + onChanged(); + } + } else { + if (!other.stripeStats_.isEmpty()) { + if (stripeStatsBuilder_.isEmpty()) { + stripeStatsBuilder_.dispose(); + stripeStatsBuilder_ = null; + stripeStats_ = other.stripeStats_; + bitField0_ = (bitField0_ & ~0x00000001); + stripeStatsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getStripeStatsFieldBuilder() : null; + } else { + stripeStatsBuilder_.addAllMessages(other.stripeStats_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + private java.util.List stripeStats_ = + java.util.Collections.emptyList(); + private void ensureStripeStatsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + stripeStats_ = new java.util.ArrayList(stripeStats_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder> stripeStatsBuilder_; + + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List getStripeStatsList() { + if (stripeStatsBuilder_ == null) { + return java.util.Collections.unmodifiableList(stripeStats_); + } else { + return stripeStatsBuilder_.getMessageList(); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public int getStripeStatsCount() { + if (stripeStatsBuilder_ == null) { + return stripeStats_.size(); + } else { + return stripeStatsBuilder_.getCount(); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics getStripeStats(int index) { + if (stripeStatsBuilder_ == null) { + return stripeStats_.get(index); + } else { + return stripeStatsBuilder_.getMessage(index); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder setStripeStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics value) { + if (stripeStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStripeStatsIsMutable(); + stripeStats_.set(index, value); + onChanged(); + } else { + stripeStatsBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder setStripeStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder builderForValue) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + stripeStats_.set(index, builderForValue.build()); + onChanged(); + } else { + stripeStatsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addStripeStats(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics value) { + if (stripeStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStripeStatsIsMutable(); + stripeStats_.add(value); + onChanged(); + } else { + stripeStatsBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addStripeStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics value) { + if (stripeStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStripeStatsIsMutable(); + stripeStats_.add(index, value); + onChanged(); + } else { + stripeStatsBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addStripeStats( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder builderForValue) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + stripeStats_.add(builderForValue.build()); + onChanged(); + } else { + stripeStatsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addStripeStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder builderForValue) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + stripeStats_.add(index, builderForValue.build()); + onChanged(); + } else { + stripeStatsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addAllStripeStats( + java.lang.Iterable values) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + super.addAll(values, stripeStats_); + onChanged(); + } else { + stripeStatsBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder clearStripeStats() { + if (stripeStatsBuilder_ == null) { + stripeStats_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + stripeStatsBuilder_.clear(); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder removeStripeStats(int index) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + stripeStats_.remove(index); + onChanged(); + } else { + stripeStatsBuilder_.remove(index); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder getStripeStatsBuilder( + int index) { + return getStripeStatsFieldBuilder().getBuilder(index); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder getStripeStatsOrBuilder( + int index) { + if (stripeStatsBuilder_ == null) { + return stripeStats_.get(index); } else { + return stripeStatsBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List + getStripeStatsOrBuilderList() { + if (stripeStatsBuilder_ != null) { + return stripeStatsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(stripeStats_); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder addStripeStatsBuilder() { + return getStripeStatsFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.getDefaultInstance()); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder addStripeStatsBuilder( + int index) { + return getStripeStatsFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.getDefaultInstance()); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List + getStripeStatsBuilderList() { + return getStripeStatsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder> + getStripeStatsFieldBuilder() { + if (stripeStatsBuilder_ == null) { + stripeStatsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder>( + stripeStats_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + stripeStats_ = null; + } + return stripeStatsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.Metadata) + } + + static { + defaultInstance = new Metadata(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.Metadata) + } + public interface FooterOrBuilder extends com.google.protobuf.MessageOrBuilder { @@ -14286,6 +15634,16 @@ public Builder clearRowIndexStride() { */ int getVersion(int index); + // optional uint64 metadataLength = 5; + /** + * optional uint64 metadataLength = 5; + */ + boolean hasMetadataLength(); + /** + * optional uint64 metadataLength = 5; + */ + long getMetadataLength(); + // optional string magic = 8000; /** * optional string magic = 8000; @@ -14410,8 +15768,13 @@ private PostScript( input.popLimit(limit); break; } - case 64002: { + case 40: { bitField0_ |= 0x00000008; + metadataLength_ = input.readUInt64(); + break; + } + case 64002: { + bitField0_ |= 0x00000010; magic_ = input.readBytes(); break; } @@ -14530,6 +15893,22 @@ public int getVersion(int index) { } private int versionMemoizedSerializedSize = -1; + // optional uint64 metadataLength = 5; + public static final int METADATALENGTH_FIELD_NUMBER = 5; + private long metadataLength_; + /** + * optional uint64 metadataLength = 5; + */ + public boolean hasMetadataLength() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional uint64 metadataLength = 5; + */ + public long getMetadataLength() { + return metadataLength_; + } + // optional string magic = 8000; public static final int MAGIC_FIELD_NUMBER = 8000; private java.lang.Object magic_; @@ -14541,7 +15920,7 @@ public int getVersion(int index) { * */ public boolean hasMagic() { - return ((bitField0_ & 0x00000008) == 0x00000008); + return ((bitField0_ & 0x00000010) == 0x00000010); } /** * optional string magic = 8000; @@ -14590,6 +15969,7 @@ private void initFields() { compression_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind.NONE; compressionBlockSize_ = 0L; version_ = java.util.Collections.emptyList(); + metadataLength_ = 0L; magic_ = ""; } private byte memoizedIsInitialized = -1; @@ -14621,6 +16001,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) output.writeUInt32NoTag(version_.get(i)); } if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeUInt64(5, metadataLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { output.writeBytes(8000, getMagicBytes()); } getUnknownFields().writeTo(output); @@ -14660,6 +16043,10 @@ public int getSerializedSize() { } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(5, metadataLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream .computeBytesSize(8000, getMagicBytes()); } size += getUnknownFields().getSerializedSize(); @@ -14790,8 +16177,10 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00000004); version_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000008); - magic_ = ""; + metadataLength_ = 0L; bitField0_ = (bitField0_ & ~0x00000010); + magic_ = ""; + bitField0_ = (bitField0_ & ~0x00000020); return this; } @@ -14840,6 +16229,10 @@ public Builder clone() { if (((from_bitField0_ & 0x00000010) == 0x00000010)) { to_bitField0_ |= 0x00000008; } + result.metadataLength_ = metadataLength_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000010; + } result.magic_ = magic_; result.bitField0_ = to_bitField0_; onBuilt(); @@ -14876,8 +16269,11 @@ public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript ot } onChanged(); } + if (other.hasMetadataLength()) { + setMetadataLength(other.getMetadataLength()); + } if (other.hasMagic()) { - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000020; magic_ = other.magic_; onChanged(); } @@ -15076,6 +16472,39 @@ public Builder clearVersion() { return this; } + // optional uint64 metadataLength = 5; + private long metadataLength_ ; + /** + * optional uint64 metadataLength = 5; + */ + public boolean hasMetadataLength() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional uint64 metadataLength = 5; + */ + public long getMetadataLength() { + return metadataLength_; + } + /** + * optional uint64 metadataLength = 5; + */ + public Builder setMetadataLength(long value) { + bitField0_ |= 0x00000010; + metadataLength_ = value; + onChanged(); + return this; + } + /** + * optional uint64 metadataLength = 5; + */ + public Builder clearMetadataLength() { + bitField0_ = (bitField0_ & ~0x00000010); + metadataLength_ = 0L; + onChanged(); + return this; + } + // optional string magic = 8000; private java.lang.Object magic_ = ""; /** @@ -15086,7 +16515,7 @@ public Builder clearVersion() { * */ public boolean hasMagic() { - return ((bitField0_ & 0x00000010) == 0x00000010); + return ((bitField0_ & 0x00000020) == 0x00000020); } /** * optional string magic = 8000; @@ -15138,7 +16567,7 @@ public Builder setMagic( if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000020; magic_ = value; onChanged(); return this; @@ -15151,7 +16580,7 @@ public Builder setMagic( * */ public Builder clearMagic() { - bitField0_ = (bitField0_ & ~0x00000010); + bitField0_ = (bitField0_ & ~0x00000020); magic_ = getDefaultInstance().getMagic(); onChanged(); return this; @@ -15168,7 +16597,7 @@ public Builder setMagicBytes( if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000020; magic_ = value; onChanged(); return this; @@ -15266,6 +16695,16 @@ public Builder setMagicBytes( com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_fieldAccessorTable; private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor; private static com.google.protobuf.GeneratedMessage.FieldAccessorTable @@ -15341,22 +16780,27 @@ public Builder setMagicBytes( "ffset\030\001 \001(\004\022\023\n\013indexLength\030\002 \001(\004\022\022\n\ndata" + "Length\030\003 \001(\004\022\024\n\014footerLength\030\004 \001(\004\022\024\n\014nu" + "mberOfRows\030\005 \001(\004\"/\n\020UserMetadataItem\022\014\n\004" + - "name\030\001 \002(\t\022\r\n\005value\030\002 \002(\014\"\356\002\n\006Footer\022\024\n\014" + - "headerLength\030\001 \001(\004\022\025\n\rcontentLength\030\002 \001(" + - "\004\022D\n\007stripes\030\003 \003(\01323.org.apache.hadoop.h", - "ive.ql.io.orc.StripeInformation\0225\n\005types" + - "\030\004 \003(\0132&.org.apache.hadoop.hive.ql.io.or" + - "c.Type\022D\n\010metadata\030\005 \003(\01322.org.apache.ha" + - "doop.hive.ql.io.orc.UserMetadataItem\022\024\n\014" + - "numberOfRows\030\006 \001(\004\022F\n\nstatistics\030\007 \003(\01322" + - ".org.apache.hadoop.hive.ql.io.orc.Column" + - "Statistics\022\026\n\016rowIndexStride\030\010 \001(\r\"\255\001\n\nP" + - "ostScript\022\024\n\014footerLength\030\001 \001(\004\022F\n\013compr" + - "ession\030\002 \001(\01621.org.apache.hadoop.hive.ql" + - ".io.orc.CompressionKind\022\034\n\024compressionBl", - "ockSize\030\003 \001(\004\022\023\n\007version\030\004 \003(\rB\002\020\001\022\016\n\005ma" + - "gic\030\300> \001(\t*:\n\017CompressionKind\022\010\n\004NONE\020\000\022" + - "\010\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003" + "name\030\001 \002(\t\022\r\n\005value\030\002 \002(\014\"X\n\020StripeStati" + + "stics\022D\n\010colStats\030\001 \003(\01322.org.apache.had" + + "oop.hive.ql.io.orc.ColumnStatistics\"S\n\010M", + "etadata\022G\n\013stripeStats\030\001 \003(\01322.org.apach" + + "e.hadoop.hive.ql.io.orc.StripeStatistics" + + "\"\356\002\n\006Footer\022\024\n\014headerLength\030\001 \001(\004\022\025\n\rcon" + + "tentLength\030\002 \001(\004\022D\n\007stripes\030\003 \003(\01323.org." + + "apache.hadoop.hive.ql.io.orc.StripeInfor" + + "mation\0225\n\005types\030\004 \003(\0132&.org.apache.hadoo" + + "p.hive.ql.io.orc.Type\022D\n\010metadata\030\005 \003(\0132" + + "2.org.apache.hadoop.hive.ql.io.orc.UserM" + + "etadataItem\022\024\n\014numberOfRows\030\006 \001(\004\022F\n\nsta" + + "tistics\030\007 \003(\01322.org.apache.hadoop.hive.q", + "l.io.orc.ColumnStatistics\022\026\n\016rowIndexStr" + + "ide\030\010 \001(\r\"\305\001\n\nPostScript\022\024\n\014footerLength" + + "\030\001 \001(\004\022F\n\013compression\030\002 \001(\01621.org.apache" + + ".hadoop.hive.ql.io.orc.CompressionKind\022\034" + + "\n\024compressionBlockSize\030\003 \001(\004\022\023\n\007version\030" + + "\004 \003(\rB\002\020\001\022\026\n\016metadataLength\030\005 \001(\004\022\016\n\005mag" + + "ic\030\300> \001(\t*:\n\017CompressionKind\022\010\n\004NONE\020\000\022\010" + + "\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -15459,18 +16903,30 @@ public Builder setMagicBytes( com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_descriptor, new java.lang.String[] { "Name", "Value", }); - internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor = + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor = getDescriptor().getMessageTypes().get(16); + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor, + new java.lang.String[] { "ColStats", }); + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor = + getDescriptor().getMessageTypes().get(17); + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor, + new java.lang.String[] { "StripeStats", }); + internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor = + getDescriptor().getMessageTypes().get(18); internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor, new java.lang.String[] { "HeaderLength", "ContentLength", "Stripes", "Types", "Metadata", "NumberOfRows", "Statistics", "RowIndexStride", }); internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor = - getDescriptor().getMessageTypes().get(17); + getDescriptor().getMessageTypes().get(19); internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor, - new java.lang.String[] { "FooterLength", "Compression", "CompressionBlockSize", "Version", "Magic", }); + new java.lang.String[] { "FooterLength", "Compression", "CompressionBlockSize", "Version", "MetadataLength", "Magic", }); return null; } }; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java index d2d9912..7a3f508 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java @@ -41,8 +41,17 @@ public static void main(String[] args) throws Exception { System.out.println("Compression size: " + reader.getCompressionSize()); } System.out.println("Type: " + reader.getObjectInspector().getTypeName()); + System.out.println("\nStripe Statistics:"); + Metadata metadata = reader.getMetadata(); + for (int n = 0; n < metadata.getStripeStatistics().size(); n++) { + System.out.println(" Stripe " + (n + 1) + ":"); + StripeStatistics ss = metadata.getStripeStatistics().get(n); + for (int i = 0; i < ss.getColumnStatistics().length; ++i) { + System.out.println(" Column " + i + ": " + ss.getColumnStatistics()[i].toString()); + } + } ColumnStatistics[] stats = reader.getStatistics(); - System.out.println("\nStatistics:"); + System.out.println("\nFile Statistics:"); for(int i=0; i < stats.length; ++i) { System.out.println(" Column " + i + ": " + stats[i].toString()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/Metadata.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/Metadata.java new file mode 100644 index 0000000..64d6d5d --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/Metadata.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.util.List; + +import com.google.common.collect.Lists; + +public class Metadata { + + private final OrcProto.Metadata metadata; + + Metadata(OrcProto.Metadata m) { + this.metadata = m; + } + + /** + * Return list of stripe level column statistics + * + * @return list of stripe statistics + */ + public List getStripeStatistics() { + List result = Lists.newArrayList(); + for (OrcProto.StripeStatistics ss : metadata.getStripeStatsList()) { + result.add(new StripeStatistics(ss.getColStatsList())); + } + return result; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java index 37d7d86..b623496 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java @@ -18,13 +18,13 @@ package org.apache.hadoop.hive.ql.io.orc; -import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + /** * The interface for reading ORC files. * @@ -108,6 +108,13 @@ ColumnStatistics[] getStatistics(); /** + * Get the metadata information like stripe level column statistics etc. + * @return the information about the column + * @throws IOException + */ + Metadata getMetadata() throws IOException; + + /** * Get the list of types contained in the file. The root type is the first * type in the list. * @return the list of flattened types @@ -134,6 +141,7 @@ * @throws IOException * @deprecated */ + @Deprecated RecordReader rows(long offset, long length, boolean[] include) throws IOException; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index 2f3d735..eadf605 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -53,6 +53,9 @@ private final CompressionKind compressionKind; private final CompressionCodec codec; private final int bufferSize; + private OrcProto.Metadata metadata = null; + private final int metadataSize; + private final int footerOffset; private final OrcProto.Footer footer; private final ObjectInspector inspector; private long deserializedSize = -1; @@ -290,6 +293,8 @@ static void checkOrcVersion(Log log, Path path, List version) { OrcProto.PostScript ps = OrcProto.PostScript.parseFrom(in); checkOrcVersion(LOG, path, ps.getVersionList()); int footerSize = (int) ps.getFooterLength(); + metadataSize = (int) ps.getMetadataLength(); + footerOffset = (int) (size - ( psLen + 1 + footerSize)); bufferSize = (int) ps.getCompressionBlockSize(); switch (ps.getCompression()) { case NONE: @@ -323,10 +328,22 @@ static void checkOrcVersion(Log log, Path path, List version) { buffer.position(psOffset - footerSize); buffer.limit(psOffset); } + // read footer InputStream instream = InStream.create("footer", new ByteBuffer[]{buffer}, new long[]{0L}, footerSize, codec, bufferSize); footer = OrcProto.Footer.parseFrom(instream); inspector = OrcStruct.createObjectInspector(0, footer.getTypesList()); + + // if metadata is already contained in first 16K file read then parse it + // else do it lazily + if(extra == 0) { + buffer.position(psOffset - (footerSize + metadataSize)); + buffer.limit(psOffset - footerSize); + instream = InStream.create("metadata", new ByteBuffer[]{buffer}, + new long[]{0L}, metadataSize, codec, bufferSize); + metadata = OrcProto.Metadata.parseFrom(instream); + } + file.close(); } @@ -470,4 +487,23 @@ private int getLastIdx() { return Collections.max(indices); } + @Override + public Metadata getMetadata() throws IOException { + // if metadata is not parsed already then read and parse it + if (metadata == null && metadataSize > 0) { + FSDataInputStream file = this.fileSystem.open(path); + file.seek(footerOffset - metadataSize); + ByteBuffer buffer = ByteBuffer.allocate(metadataSize); + file.readFully(buffer.array(), buffer.arrayOffset() + buffer.position(), + buffer.remaining()); + buffer.position(0); + buffer.limit(metadataSize); + InputStream instream = InStream.create("metadata", new ByteBuffer[] {buffer}, + new long[] {0L}, metadataSize, codec, bufferSize); + metadata = OrcProto.Metadata.parseFrom(instream); + file.close(); + } + return new Metadata(metadata); + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java new file mode 100644 index 0000000..6fc1ab6 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.util.List; + +public class StripeStatistics { + private final List cs; + + StripeStatistics(List list) { + this.cs = list; + } + + /** + * Return list of column statistics + * + * @return column stats + */ + public ColumnStatistics[] getColumnStatistics() { + ColumnStatistics[] result = new ColumnStatistics[cs.size()]; + for (int i = 0; i < result.length; ++i) { + result[i] = ColumnStatisticsImpl.deserialize(cs.get(i)); + } + return result; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java index 76b7cf3..4658049 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry; +import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics; import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type; import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.hive.serde2.io.DateWritable; @@ -64,6 +65,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; +import com.google.common.collect.Lists; import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; @@ -125,7 +127,6 @@ private final boolean buildIndex; private final MemoryManager memoryManager; private final OrcFile.Version version; - private final Configuration conf; WriterImpl(FileSystem fs, @@ -411,6 +412,7 @@ public Configuration getConfiguration() { private final BitFieldWriter isPresent; private final boolean isCompressed; protected final ColumnStatisticsImpl indexStatistics; + protected final ColumnStatisticsImpl stripeColStatistics; private final ColumnStatisticsImpl fileStatistics; protected TreeWriter[] childrenWriters; protected final RowIndexPositionRecorder rowIndexPosition; @@ -419,7 +421,7 @@ public Configuration getConfiguration() { private final PositionedOutputStream rowIndexStream; private boolean foundNulls; private OutStream isPresentOutStream; - protected final boolean useDirectV2Encoding; + private final List stripeStatsBuilders; /** * Create a tree writer. @@ -435,7 +437,6 @@ public Configuration getConfiguration() { this.isCompressed = streamFactory.isCompressed(); this.id = columnId; this.inspector = inspector; - this.useDirectV2Encoding = true; if (nullable) { isPresentOutStream = streamFactory.createStream(id, OrcProto.Stream.Kind.PRESENT); @@ -445,11 +446,13 @@ public Configuration getConfiguration() { } this.foundNulls = false; indexStatistics = ColumnStatisticsImpl.create(inspector); + stripeColStatistics = ColumnStatisticsImpl.create(inspector); fileStatistics = ColumnStatisticsImpl.create(inspector); childrenWriters = new TreeWriter[0]; rowIndex = OrcProto.RowIndex.newBuilder(); rowIndexEntry = OrcProto.RowIndexEntry.newBuilder(); rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry); + stripeStatsBuilders = Lists.newArrayList(); if (streamFactory.buildIndex()) { rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Kind.ROW_INDEX); @@ -462,6 +465,10 @@ public Configuration getConfiguration() { return rowIndex; } + protected ColumnStatisticsImpl getStripeStatistics() { + return stripeColStatistics; + } + protected ColumnStatisticsImpl getFileStatistics() { return fileStatistics; } @@ -537,6 +544,12 @@ void writeStripe(OrcProto.StripeFooter.Builder builder, } } + // merge stripe-level column statistics to file statistics and write it to + // stripe statistics + OrcProto.StripeStatistics.Builder stripeStatsBuilder = OrcProto.StripeStatistics.newBuilder(); + writeStripeStatistics(stripeStatsBuilder, this); + stripeStatsBuilders.add(stripeStatsBuilder); + // reset the flag for next stripe foundNulls = false; @@ -554,6 +567,16 @@ void writeStripe(OrcProto.StripeFooter.Builder builder, rowIndexEntry.clear(); } + private void writeStripeStatistics(OrcProto.StripeStatistics.Builder builder, + TreeWriter treeWriter) { + treeWriter.fileStatistics.merge(treeWriter.stripeColStatistics); + builder.addColStats(treeWriter.stripeColStatistics.serialize().build()); + treeWriter.stripeColStatistics.reset(); + for (TreeWriter child : treeWriter.getChildrenWriters()) { + writeStripeStatistics(builder, child); + } + } + TreeWriter[] getChildrenWriters() { return childrenWriters; } @@ -575,7 +598,7 @@ void writeStripe(OrcProto.StripeFooter.Builder builder, * @throws IOException */ void createRowIndexEntry() throws IOException { - fileStatistics.merge(indexStatistics); + stripeColStatistics.merge(indexStatistics); rowIndexEntry.setStatistics(indexStatistics.serialize()); indexStatistics.reset(); rowIndex.addEntry(rowIndexEntry); @@ -1013,7 +1036,7 @@ public void visit(StringRedBlackTree.VisitorContext context */ @Override void createRowIndexEntry() throws IOException { - getFileStatistics().merge(indexStatistics); + getStripeStatistics().merge(indexStatistics); OrcProto.RowIndexEntry.Builder rowIndexEntry = getRowIndexEntry(); rowIndexEntry.setStatistics(indexStatistics.serialize()); indexStatistics.reset(); @@ -1851,6 +1874,21 @@ private void writeFileStatistics(OrcProto.Footer.Builder builder, } } + private int writeMetadata(long bodyLength) throws IOException { + ensureWriter(); + OrcProto.Metadata.Builder builder = OrcProto.Metadata.newBuilder(); + for(OrcProto.StripeStatistics.Builder ssb : treeWriter.stripeStatsBuilders) { + builder.addStripeStats(ssb.build()); + } + + long startPosn = rawWriter.getPos(); + OrcProto.Metadata metadata = builder.build(); + metadata.writeTo(protobufWriter); + protobufWriter.flush(); + writer.flush(); + return (int) (rawWriter.getPos() - startPosn); + } + private int writeFooter(long bodyLength) throws IOException { ensureWriter(); OrcProto.Footer.Builder builder = OrcProto.Footer.newBuilder(); @@ -1881,11 +1919,12 @@ private int writeFooter(long bodyLength) throws IOException { return (int) (rawWriter.getPos() - startPosn); } - private int writePostScript(int footerLength) throws IOException { + private int writePostScript(int footerLength, int metadataLength) throws IOException { OrcProto.PostScript.Builder builder = OrcProto.PostScript.newBuilder() .setCompression(writeCompressionKind(compress)) .setFooterLength(footerLength) + .setMetadataLength(metadataLength) .setMagic(OrcFile.MAGIC) .addVersion(version.getMajor()) .addVersion(version.getMinor()); @@ -1940,8 +1979,9 @@ public void close() throws IOException { // actually close the file synchronized (this) { flushStripe(); - int footerLength = writeFooter(rawWriter.getPos()); - rawWriter.writeByte(writePostScript(footerLength)); + int metadataLength = writeMetadata(rawWriter.getPos()); + int footerLength = writeFooter(rawWriter.getPos() - metadataLength); + rawWriter.writeByte(writePostScript(footerLength, metadataLength)); rawWriter.close(); } } diff --git ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto index 2f20793..89d3763 100644 --- ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto +++ ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto @@ -152,6 +152,14 @@ message UserMetadataItem { required bytes value = 2; } +message StripeStatistics { + repeated ColumnStatistics colStats = 1; +} + +message Metadata { + repeated StripeStatistics stripeStats = 1; +} + message Footer { optional uint64 headerLength = 1; optional uint64 contentLength = 2; @@ -176,6 +184,7 @@ message PostScript { optional CompressionKind compression = 2; optional uint64 compressionBlockSize = 3; repeated uint32 version = 4 [packed = true]; + optional uint64 metadataLength = 5; // Leave this last in the record optional string magic = 8000; } diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java index c03a80a..e857903 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java @@ -539,6 +539,70 @@ public void testStringAndBinaryStatistics() throws Exception { rows.close(); } + + @Test + public void testStripeLevelStats() throws Exception { + ObjectInspector inspector; + synchronized (TestOrcFile.class) { + inspector = ObjectInspectorFactory.getReflectionObjectInspector + (InnerStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + } + Writer writer = OrcFile.createWriter(testFilePath, + OrcFile.writerOptions(conf) + .inspector(inspector) + .stripeSize(100000) + .bufferSize(10000)); + for (int i = 0; i < 11000; i++) { + if (i >= 5000) { + if (i >= 10000) { + writer.addRow(new InnerStruct(3, "three")); + } else { + writer.addRow(new InnerStruct(2, "two")); + } + } else { + writer.addRow(new InnerStruct(1, "one")); + } + } + + writer.close(); + Reader reader = OrcFile.createReader(fs, testFilePath); + Metadata metadata = reader.getMetadata(); + int numStripes = metadata.getStripeStatistics().size(); + assertEquals(3, numStripes); + StripeStatistics ss1 = metadata.getStripeStatistics().get(0); + StripeStatistics ss2 = metadata.getStripeStatistics().get(1); + StripeStatistics ss3 = metadata.getStripeStatistics().get(2); + assertEquals(4996, ss1.getColumnStatistics()[0].getNumberOfValues()); + assertEquals(5000, ss2.getColumnStatistics()[0].getNumberOfValues()); + assertEquals(1004, ss3.getColumnStatistics()[0].getNumberOfValues()); + + assertEquals(4996, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getNumberOfValues()); + assertEquals(5000, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getNumberOfValues()); + assertEquals(1004, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getNumberOfValues()); + assertEquals(1, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getMinimum()); + assertEquals(1, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getMinimum()); + assertEquals(2, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getMinimum()); + assertEquals(1, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getMaximum()); + assertEquals(2, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getMaximum()); + assertEquals(3, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getMaximum()); + assertEquals(4996, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getSum()); + assertEquals(9996, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getSum()); + assertEquals(3008, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getSum()); + + assertEquals(4996, ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getNumberOfValues()); + assertEquals(5000, ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getNumberOfValues()); + assertEquals(1004, ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getNumberOfValues()); + assertEquals("one", ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getMinimum()); + assertEquals("one", ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getMinimum()); + assertEquals("three", ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getMinimum()); + assertEquals("one", ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getMaximum()); + assertEquals("two", ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getMaximum()); + assertEquals("two", ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getMaximum()); + assertEquals(14988, ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getSum()); + assertEquals(15000, ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getSum()); + assertEquals(5012, ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getSum()); + } + @Test public void test1() throws Exception { ObjectInspector inspector; @@ -547,12 +611,12 @@ public void test1() throws Exception { (BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } Writer writer = OrcFile.createWriter(testFilePath, - OrcFile.writerOptions(conf) - .inspector(inspector) - .stripeSize(100000) - .bufferSize(10000)); + OrcFile.writerOptions(conf) + .inspector(inspector) + .stripeSize(100000) + .bufferSize(10000)); writer.addRow(new BigRow(false, (byte) 1, (short) 1024, 65536, - Long.MAX_VALUE, (float) 1.0, -15.0, bytes(0,1,2,3,4), "hi", + Long.MAX_VALUE, (float) 1.0, -15.0, bytes(0, 1, 2, 3, 4), "hi", new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), list(inner(3, "good"), inner(4, "bad")), map())); @@ -560,10 +624,12 @@ public void test1() throws Exception { Long.MAX_VALUE, (float) 2.0, -5.0, bytes(), "bye", new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), list(inner(100000000, "cat"), inner(-100000, "in"), inner(1234, "hat")), - map(inner(5,"chani"), inner(1,"mauddib")))); + map(inner(5, "chani"), inner(1, "mauddib")))); writer.close(); Reader reader = OrcFile.createReader(fs, testFilePath); + Metadata metadata = reader.getMetadata(); + // check the stats ColumnStatistics[] stats = reader.getStatistics(); assertEquals(2, stats[1].getNumberOfValues()); @@ -578,14 +644,12 @@ public void test1() throws Exception { assertEquals("count: 2 min: 1024 max: 2048 sum: 3072", stats[3].toString()); - assertEquals(Long.MAX_VALUE, - ((IntegerColumnStatistics) stats[5]).getMaximum()); - assertEquals(Long.MAX_VALUE, - ((IntegerColumnStatistics) stats[5]).getMinimum()); - assertEquals(false, ((IntegerColumnStatistics) stats[5]).isSumDefined()); - assertEquals("count: 2 min: 9223372036854775807 max: 9223372036854775807", - stats[5].toString()); - + StripeStatistics ss = metadata.getStripeStatistics().get(0); + assertEquals(2, ss.getColumnStatistics()[0].getNumberOfValues()); + assertEquals(1, ((BooleanColumnStatistics) ss.getColumnStatistics()[1]).getTrueCount()); + assertEquals(1024, ((IntegerColumnStatistics) ss.getColumnStatistics()[3]).getMinimum()); + assertEquals(2048, ((IntegerColumnStatistics) ss.getColumnStatistics()[3]).getMaximum()); + assertEquals(3072, ((IntegerColumnStatistics) ss.getColumnStatistics()[3]).getSum()); assertEquals(-15.0, ((DoubleColumnStatistics) stats[7]).getMinimum()); assertEquals(-5.0, ((DoubleColumnStatistics) stats[7]).getMaximum()); assertEquals(-20.0, ((DoubleColumnStatistics) stats[7]).getSum(), 0.00001); @@ -935,6 +999,9 @@ public void metaData() throws Exception { } } assertEquals(3, i); + Metadata metadata = reader.getMetadata(); + int numStripes = metadata.getStripeStatistics().size(); + assertEquals(1, numStripes); } /** diff --git ql/src/test/resources/orc-file-dump-dictionary-threshold.out ql/src/test/resources/orc-file-dump-dictionary-threshold.out index bac7465..8687b26 100644 --- ql/src/test/resources/orc-file-dump-dictionary-threshold.out +++ ql/src/test/resources/orc-file-dump-dictionary-threshold.out @@ -4,7 +4,34 @@ Compression: ZLIB Compression size: 10000 Type: struct -Statistics: +Stripe Statistics: + Stripe 1: + Column 0: count: 4000 + Column 1: count: 4000 min: -2147115959 max: 2145911404 sum: 71315665983 + Column 2: count: 4000 min: -9211329013123260308 max: 9217851628057711416 + Column 3: count: 4000 min: Darkness,-230 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788 sum: 245096 + Stripe 2: + Column 0: count: 5000 + Column 1: count: 5000 min: -2147390285 max: 2146838901 sum: 107869424275 + Column 2: count: 5000 min: -9222178666167296739 max: 9221301751385928177 + Column 3: count: 5000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984 sum: 972748 + Stripe 3: + Column 0: count: 5000 + Column 1: count: 5000 min: -2145928262 max: 2147224606 sum: 38276585043 + Column 2: count: 5000 min: -9221963099397084326 max: 9222722740629726770 + Column 3: count: 5000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766 sum: 1753024 + Stripe 4: + Column 0: count: 5000 + Column 1: count: 5000 min: -2145378214 max: 2147453086 sum: -43469576640 + Column 2: count: 5000 min: -9222731174895935707 max: 9222919052987871506 + Column 3: count: 5000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788 sum: 2636664 + Stripe 5: + Column 0: count: 2000 + Column 1: count: 2000 min: -2143595397 max: 2144595861 sum: -64863580335 + Column 2: count: 2000 min: -9212379634781416464 max: 9208134757538374043 + Column 3: count: 2000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214-18444-18446-18724-18912-18952-19164 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 sum: 1302706 + +File Statistics: Column 0: count: 21000 Column 1: count: 21000 min: -2147390285 max: 2147453086 sum: 109128518326 Column 2: count: 21000 min: -9222731174895935707 max: 9222919052987871506 diff --git ql/src/test/resources/orc-file-dump.out ql/src/test/resources/orc-file-dump.out index 5b5eb2c..7a5b907 100644 --- ql/src/test/resources/orc-file-dump.out +++ ql/src/test/resources/orc-file-dump.out @@ -4,7 +4,34 @@ Compression: ZLIB Compression size: 10000 Type: struct -Statistics: +Stripe Statistics: + Stripe 1: + Column 0: count: 5000 + Column 1: count: 5000 min: -2146021688 max: 2147223299 sum: 515792826 + Column 2: count: 5000 min: -9218592812243954469 max: 9221614132680747961 + Column 3: count: 5000 min: Darkness, max: worst sum: 19280 + Stripe 2: + Column 0: count: 5000 + Column 1: count: 5000 min: -2146733128 max: 2147001622 sum: 7673427 + Column 2: count: 5000 min: -9220818777591257749 max: 9222259462014003839 + Column 3: count: 5000 min: Darkness, max: worst sum: 19504 + Stripe 3: + Column 0: count: 5000 + Column 1: count: 5000 min: -2146993718 max: 2147378179 sum: 132660742551 + Column 2: count: 5000 min: -9218342074710552826 max: 9222303228623055266 + Column 3: count: 5000 min: Darkness, max: worst sum: 19641 + Stripe 4: + Column 0: count: 5000 + Column 1: count: 5000 min: -2146658006 max: 2145520931 sum: 8533549236 + Column 2: count: 5000 min: -9222758097219661129 max: 9221043130193737406 + Column 3: count: 5000 min: Darkness, max: worst sum: 19470 + Stripe 5: + Column 0: count: 1000 + Column 1: count: 1000 min: -2146245500 max: 2146378640 sum: 51299706363 + Column 2: count: 1000 min: -9208193203370316142 max: 9218567213558056476 + Column 3: count: 1000 min: Darkness, max: worst sum: 3866 + +File Statistics: Column 0: count: 21000 Column 1: count: 21000 min: -2146993718 max: 2147378179 sum: 193017464403 Column 2: count: 21000 min: -9222758097219661129 max: 9222303228623055266