diff --git ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java index d33d66e..ebc2437 100644 --- ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java +++ ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java @@ -11966,6 +11966,1354 @@ public Builder clearValue() { // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.UserMetadataItem) } + public interface StripeStatisticsOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + java.util.List + getColStatsList(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getColStats(int index); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + int getColStatsCount(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + java.util.List + getColStatsOrBuilderList(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getColStatsOrBuilder( + int index); + } + /** + * Protobuf type {@code org.apache.hadoop.hive.ql.io.orc.StripeStatistics} + */ + public static final class StripeStatistics extends + com.google.protobuf.GeneratedMessage + implements StripeStatisticsOrBuilder { + // Use StripeStatistics.newBuilder() to construct. + private StripeStatistics(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private StripeStatistics(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final StripeStatistics defaultInstance; + public static StripeStatistics getDefaultInstance() { + return defaultInstance; + } + + public StripeStatistics getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private StripeStatistics( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + colStats_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000001; + } + colStats_.add(input.readMessage(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.PARSER, extensionRegistry)); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + colStats_ = java.util.Collections.unmodifiableList(colStats_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public StripeStatistics parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new StripeStatistics(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + public static final int COLSTATS_FIELD_NUMBER = 1; + private java.util.List colStats_; + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List getColStatsList() { + return colStats_; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List + getColStatsOrBuilderList() { + return colStats_; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public int getColStatsCount() { + return colStats_.size(); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getColStats(int index) { + return colStats_.get(index); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getColStatsOrBuilder( + int index) { + return colStats_.get(index); + } + + private void initFields() { + colStats_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < colStats_.size(); i++) { + output.writeMessage(1, colStats_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < colStats_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, colStats_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.apache.hadoop.hive.ql.io.orc.StripeStatistics} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder.class); + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getColStatsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (colStatsBuilder_ == null) { + colStats_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + colStatsBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics(this); + int from_bitField0_ = bitField0_; + if (colStatsBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + colStats_ = java.util.Collections.unmodifiableList(colStats_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.colStats_ = colStats_; + } else { + result.colStats_ = colStatsBuilder_.build(); + } + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.getDefaultInstance()) return this; + if (colStatsBuilder_ == null) { + if (!other.colStats_.isEmpty()) { + if (colStats_.isEmpty()) { + colStats_ = other.colStats_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureColStatsIsMutable(); + colStats_.addAll(other.colStats_); + } + onChanged(); + } + } else { + if (!other.colStats_.isEmpty()) { + if (colStatsBuilder_.isEmpty()) { + colStatsBuilder_.dispose(); + colStatsBuilder_ = null; + colStats_ = other.colStats_; + bitField0_ = (bitField0_ & ~0x00000001); + colStatsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getColStatsFieldBuilder() : null; + } else { + colStatsBuilder_.addAllMessages(other.colStats_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + private java.util.List colStats_ = + java.util.Collections.emptyList(); + private void ensureColStatsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + colStats_ = new java.util.ArrayList(colStats_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder> colStatsBuilder_; + + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List getColStatsList() { + if (colStatsBuilder_ == null) { + return java.util.Collections.unmodifiableList(colStats_); + } else { + return colStatsBuilder_.getMessageList(); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public int getColStatsCount() { + if (colStatsBuilder_ == null) { + return colStats_.size(); + } else { + return colStatsBuilder_.getCount(); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics getColStats(int index) { + if (colStatsBuilder_ == null) { + return colStats_.get(index); + } else { + return colStatsBuilder_.getMessage(index); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder setColStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (colStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureColStatsIsMutable(); + colStats_.set(index, value); + onChanged(); + } else { + colStatsBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder setColStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + colStats_.set(index, builderForValue.build()); + onChanged(); + } else { + colStatsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addColStats(org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (colStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureColStatsIsMutable(); + colStats_.add(value); + onChanged(); + } else { + colStatsBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addColStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics value) { + if (colStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureColStatsIsMutable(); + colStats_.add(index, value); + onChanged(); + } else { + colStatsBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addColStats( + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + colStats_.add(builderForValue.build()); + onChanged(); + } else { + colStatsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addColStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder builderForValue) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + colStats_.add(index, builderForValue.build()); + onChanged(); + } else { + colStatsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder addAllColStats( + java.lang.Iterable values) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + super.addAll(values, colStats_); + onChanged(); + } else { + colStatsBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder clearColStats() { + if (colStatsBuilder_ == null) { + colStats_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + colStatsBuilder_.clear(); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public Builder removeColStats(int index) { + if (colStatsBuilder_ == null) { + ensureColStatsIsMutable(); + colStats_.remove(index); + onChanged(); + } else { + colStatsBuilder_.remove(index); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder getColStatsBuilder( + int index) { + return getColStatsFieldBuilder().getBuilder(index); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder getColStatsOrBuilder( + int index) { + if (colStatsBuilder_ == null) { + return colStats_.get(index); } else { + return colStatsBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List + getColStatsOrBuilderList() { + if (colStatsBuilder_ != null) { + return colStatsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(colStats_); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder addColStatsBuilder() { + return getColStatsFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance()); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder addColStatsBuilder( + int index) { + return getColStatsFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.getDefaultInstance()); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.ColumnStatistics colStats = 1; + */ + public java.util.List + getColStatsBuilderList() { + return getColStatsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder> + getColStatsFieldBuilder() { + if (colStatsBuilder_ == null) { + colStatsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.ColumnStatisticsOrBuilder>( + colStats_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + colStats_ = null; + } + return colStatsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.StripeStatistics) + } + + static { + defaultInstance = new StripeStatistics(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.StripeStatistics) + } + + public interface MetadataOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + java.util.List + getStripeStatsList(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics getStripeStats(int index); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + int getStripeStatsCount(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + java.util.List + getStripeStatsOrBuilderList(); + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder getStripeStatsOrBuilder( + int index); + } + /** + * Protobuf type {@code org.apache.hadoop.hive.ql.io.orc.Metadata} + */ + public static final class Metadata extends + com.google.protobuf.GeneratedMessage + implements MetadataOrBuilder { + // Use Metadata.newBuilder() to construct. + private Metadata(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private Metadata(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final Metadata defaultInstance; + public static Metadata getDefaultInstance() { + return defaultInstance; + } + + public Metadata getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private Metadata( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + stripeStats_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000001; + } + stripeStats_.add(input.readMessage(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.PARSER, extensionRegistry)); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + stripeStats_ = java.util.Collections.unmodifiableList(stripeStats_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public Metadata parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new Metadata(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + // repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + public static final int STRIPESTATS_FIELD_NUMBER = 1; + private java.util.List stripeStats_; + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List getStripeStatsList() { + return stripeStats_; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List + getStripeStatsOrBuilderList() { + return stripeStats_; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public int getStripeStatsCount() { + return stripeStats_.size(); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics getStripeStats(int index) { + return stripeStats_.get(index); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder getStripeStatsOrBuilder( + int index) { + return stripeStats_.get(index); + } + + private void initFields() { + stripeStats_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < stripeStats_.size(); i++) { + output.writeMessage(1, stripeStats_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < stripeStats_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, stripeStats_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.apache.hadoop.hive.ql.io.orc.Metadata} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hive.ql.io.orc.OrcProto.MetadataOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.class, org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.Builder.class); + } + + // Construct using org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getStripeStatsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (stripeStatsBuilder_ == null) { + stripeStats_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + stripeStatsBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata getDefaultInstanceForType() { + return org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.getDefaultInstance(); + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata build() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata buildPartial() { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata result = new org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata(this); + int from_bitField0_ = bitField0_; + if (stripeStatsBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + stripeStats_ = java.util.Collections.unmodifiableList(stripeStats_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.stripeStats_ = stripeStats_; + } else { + result.stripeStats_ = stripeStatsBuilder_.build(); + } + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata) { + return mergeFrom((org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata other) { + if (other == org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata.getDefaultInstance()) return this; + if (stripeStatsBuilder_ == null) { + if (!other.stripeStats_.isEmpty()) { + if (stripeStats_.isEmpty()) { + stripeStats_ = other.stripeStats_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureStripeStatsIsMutable(); + stripeStats_.addAll(other.stripeStats_); + } + onChanged(); + } + } else { + if (!other.stripeStats_.isEmpty()) { + if (stripeStatsBuilder_.isEmpty()) { + stripeStatsBuilder_.dispose(); + stripeStatsBuilder_ = null; + stripeStats_ = other.stripeStats_; + bitField0_ = (bitField0_ & ~0x00000001); + stripeStatsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getStripeStatsFieldBuilder() : null; + } else { + stripeStatsBuilder_.addAllMessages(other.stripeStats_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hive.ql.io.orc.OrcProto.Metadata) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + private java.util.List stripeStats_ = + java.util.Collections.emptyList(); + private void ensureStripeStatsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + stripeStats_ = new java.util.ArrayList(stripeStats_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder> stripeStatsBuilder_; + + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List getStripeStatsList() { + if (stripeStatsBuilder_ == null) { + return java.util.Collections.unmodifiableList(stripeStats_); + } else { + return stripeStatsBuilder_.getMessageList(); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public int getStripeStatsCount() { + if (stripeStatsBuilder_ == null) { + return stripeStats_.size(); + } else { + return stripeStatsBuilder_.getCount(); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics getStripeStats(int index) { + if (stripeStatsBuilder_ == null) { + return stripeStats_.get(index); + } else { + return stripeStatsBuilder_.getMessage(index); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder setStripeStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics value) { + if (stripeStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStripeStatsIsMutable(); + stripeStats_.set(index, value); + onChanged(); + } else { + stripeStatsBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder setStripeStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder builderForValue) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + stripeStats_.set(index, builderForValue.build()); + onChanged(); + } else { + stripeStatsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addStripeStats(org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics value) { + if (stripeStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStripeStatsIsMutable(); + stripeStats_.add(value); + onChanged(); + } else { + stripeStatsBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addStripeStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics value) { + if (stripeStatsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureStripeStatsIsMutable(); + stripeStats_.add(index, value); + onChanged(); + } else { + stripeStatsBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addStripeStats( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder builderForValue) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + stripeStats_.add(builderForValue.build()); + onChanged(); + } else { + stripeStatsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addStripeStats( + int index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder builderForValue) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + stripeStats_.add(index, builderForValue.build()); + onChanged(); + } else { + stripeStatsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder addAllStripeStats( + java.lang.Iterable values) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + super.addAll(values, stripeStats_); + onChanged(); + } else { + stripeStatsBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder clearStripeStats() { + if (stripeStatsBuilder_ == null) { + stripeStats_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + stripeStatsBuilder_.clear(); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public Builder removeStripeStats(int index) { + if (stripeStatsBuilder_ == null) { + ensureStripeStatsIsMutable(); + stripeStats_.remove(index); + onChanged(); + } else { + stripeStatsBuilder_.remove(index); + } + return this; + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder getStripeStatsBuilder( + int index) { + return getStripeStatsFieldBuilder().getBuilder(index); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder getStripeStatsOrBuilder( + int index) { + if (stripeStatsBuilder_ == null) { + return stripeStats_.get(index); } else { + return stripeStatsBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List + getStripeStatsOrBuilderList() { + if (stripeStatsBuilder_ != null) { + return stripeStatsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(stripeStats_); + } + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder addStripeStatsBuilder() { + return getStripeStatsFieldBuilder().addBuilder( + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.getDefaultInstance()); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder addStripeStatsBuilder( + int index) { + return getStripeStatsFieldBuilder().addBuilder( + index, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.getDefaultInstance()); + } + /** + * repeated .org.apache.hadoop.hive.ql.io.orc.StripeStatistics stripeStats = 1; + */ + public java.util.List + getStripeStatsBuilderList() { + return getStripeStatsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder> + getStripeStatsFieldBuilder() { + if (stripeStatsBuilder_ == null) { + stripeStatsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics.Builder, org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatisticsOrBuilder>( + stripeStats_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + stripeStats_ = null; + } + return stripeStatsBuilder_; + } + + // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.ql.io.orc.Metadata) + } + + static { + defaultInstance = new Metadata(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.ql.io.orc.Metadata) + } + public interface FooterOrBuilder extends com.google.protobuf.MessageOrBuilder { @@ -14124,6 +15472,16 @@ public Builder clearRowIndexStride() { */ int getVersion(int index); + // optional uint64 metadataLength = 5; + /** + * optional uint64 metadataLength = 5; + */ + boolean hasMetadataLength(); + /** + * optional uint64 metadataLength = 5; + */ + long getMetadataLength(); + // optional string magic = 8000; /** * optional string magic = 8000; @@ -14248,8 +15606,13 @@ private PostScript( input.popLimit(limit); break; } - case 64002: { + case 40: { bitField0_ |= 0x00000008; + metadataLength_ = input.readUInt64(); + break; + } + case 64002: { + bitField0_ |= 0x00000010; magic_ = input.readBytes(); break; } @@ -14368,6 +15731,22 @@ public int getVersion(int index) { } private int versionMemoizedSerializedSize = -1; + // optional uint64 metadataLength = 5; + public static final int METADATALENGTH_FIELD_NUMBER = 5; + private long metadataLength_; + /** + * optional uint64 metadataLength = 5; + */ + public boolean hasMetadataLength() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional uint64 metadataLength = 5; + */ + public long getMetadataLength() { + return metadataLength_; + } + // optional string magic = 8000; public static final int MAGIC_FIELD_NUMBER = 8000; private java.lang.Object magic_; @@ -14379,7 +15758,7 @@ public int getVersion(int index) { * */ public boolean hasMagic() { - return ((bitField0_ & 0x00000008) == 0x00000008); + return ((bitField0_ & 0x00000010) == 0x00000010); } /** * optional string magic = 8000; @@ -14428,6 +15807,7 @@ private void initFields() { compression_ = org.apache.hadoop.hive.ql.io.orc.OrcProto.CompressionKind.NONE; compressionBlockSize_ = 0L; version_ = java.util.Collections.emptyList(); + metadataLength_ = 0L; magic_ = ""; } private byte memoizedIsInitialized = -1; @@ -14459,6 +15839,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) output.writeUInt32NoTag(version_.get(i)); } if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeUInt64(5, metadataLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { output.writeBytes(8000, getMagicBytes()); } getUnknownFields().writeTo(output); @@ -14498,6 +15881,10 @@ public int getSerializedSize() { } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(5, metadataLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream .computeBytesSize(8000, getMagicBytes()); } size += getUnknownFields().getSerializedSize(); @@ -14628,8 +16015,10 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00000004); version_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000008); - magic_ = ""; + metadataLength_ = 0L; bitField0_ = (bitField0_ & ~0x00000010); + magic_ = ""; + bitField0_ = (bitField0_ & ~0x00000020); return this; } @@ -14678,6 +16067,10 @@ public Builder clone() { if (((from_bitField0_ & 0x00000010) == 0x00000010)) { to_bitField0_ |= 0x00000008; } + result.metadataLength_ = metadataLength_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000010; + } result.magic_ = magic_; result.bitField0_ = to_bitField0_; onBuilt(); @@ -14714,8 +16107,11 @@ public Builder mergeFrom(org.apache.hadoop.hive.ql.io.orc.OrcProto.PostScript ot } onChanged(); } + if (other.hasMetadataLength()) { + setMetadataLength(other.getMetadataLength()); + } if (other.hasMagic()) { - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000020; magic_ = other.magic_; onChanged(); } @@ -14914,6 +16310,39 @@ public Builder clearVersion() { return this; } + // optional uint64 metadataLength = 5; + private long metadataLength_ ; + /** + * optional uint64 metadataLength = 5; + */ + public boolean hasMetadataLength() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional uint64 metadataLength = 5; + */ + public long getMetadataLength() { + return metadataLength_; + } + /** + * optional uint64 metadataLength = 5; + */ + public Builder setMetadataLength(long value) { + bitField0_ |= 0x00000010; + metadataLength_ = value; + onChanged(); + return this; + } + /** + * optional uint64 metadataLength = 5; + */ + public Builder clearMetadataLength() { + bitField0_ = (bitField0_ & ~0x00000010); + metadataLength_ = 0L; + onChanged(); + return this; + } + // optional string magic = 8000; private java.lang.Object magic_ = ""; /** @@ -14924,7 +16353,7 @@ public Builder clearVersion() { * */ public boolean hasMagic() { - return ((bitField0_ & 0x00000010) == 0x00000010); + return ((bitField0_ & 0x00000020) == 0x00000020); } /** * optional string magic = 8000; @@ -14976,7 +16405,7 @@ public Builder setMagic( if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000020; magic_ = value; onChanged(); return this; @@ -14989,7 +16418,7 @@ public Builder setMagic( * */ public Builder clearMagic() { - bitField0_ = (bitField0_ & ~0x00000010); + bitField0_ = (bitField0_ & ~0x00000020); magic_ = getDefaultInstance().getMagic(); onChanged(); return this; @@ -15006,7 +16435,7 @@ public Builder setMagicBytes( if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000020; magic_ = value; onChanged(); return this; @@ -15104,6 +16533,16 @@ public Builder setMagicBytes( com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_fieldAccessorTable; private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor; private static com.google.protobuf.GeneratedMessage.FieldAccessorTable @@ -15178,22 +16617,27 @@ public Builder setMagicBytes( "n\022\016\n\006offset\030\001 \001(\004\022\023\n\013indexLength\030\002 \001(\004\022\022" + "\n\ndataLength\030\003 \001(\004\022\024\n\014footerLength\030\004 \001(\004" + "\022\024\n\014numberOfRows\030\005 \001(\004\"/\n\020UserMetadataIt" + - "em\022\014\n\004name\030\001 \002(\t\022\r\n\005value\030\002 \002(\014\"\356\002\n\006Foot" + - "er\022\024\n\014headerLength\030\001 \001(\004\022\025\n\rcontentLengt" + - "h\030\002 \001(\004\022D\n\007stripes\030\003 \003(\01323.org.apache.ha" + - "doop.hive.ql.io.orc.StripeInformation\0225\n", - "\005types\030\004 \003(\0132&.org.apache.hadoop.hive.ql" + - ".io.orc.Type\022D\n\010metadata\030\005 \003(\01322.org.apa" + - "che.hadoop.hive.ql.io.orc.UserMetadataIt" + - "em\022\024\n\014numberOfRows\030\006 \001(\004\022F\n\nstatistics\030\007" + - " \003(\01322.org.apache.hadoop.hive.ql.io.orc." + - "ColumnStatistics\022\026\n\016rowIndexStride\030\010 \001(\r" + - "\"\255\001\n\nPostScript\022\024\n\014footerLength\030\001 \001(\004\022F\n" + - "\013compression\030\002 \001(\01621.org.apache.hadoop.h" + - "ive.ql.io.orc.CompressionKind\022\034\n\024compres" + - "sionBlockSize\030\003 \001(\004\022\023\n\007version\030\004 \003(\rB\002\020\001", - "\022\016\n\005magic\030\300> \001(\t*:\n\017CompressionKind\022\010\n\004N" + - "ONE\020\000\022\010\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003" + "em\022\014\n\004name\030\001 \002(\t\022\r\n\005value\030\002 \002(\014\"X\n\020Strip" + + "eStatistics\022D\n\010colStats\030\001 \003(\01322.org.apac" + + "he.hadoop.hive.ql.io.orc.ColumnStatistic" + + "s\"S\n\010Metadata\022G\n\013stripeStats\030\001 \003(\01322.org", + ".apache.hadoop.hive.ql.io.orc.StripeStat" + + "istics\"\356\002\n\006Footer\022\024\n\014headerLength\030\001 \001(\004\022" + + "\025\n\rcontentLength\030\002 \001(\004\022D\n\007stripes\030\003 \003(\0132" + + "3.org.apache.hadoop.hive.ql.io.orc.Strip" + + "eInformation\0225\n\005types\030\004 \003(\0132&.org.apache" + + ".hadoop.hive.ql.io.orc.Type\022D\n\010metadata\030" + + "\005 \003(\01322.org.apache.hadoop.hive.ql.io.orc" + + ".UserMetadataItem\022\024\n\014numberOfRows\030\006 \001(\004\022" + + "F\n\nstatistics\030\007 \003(\01322.org.apache.hadoop." + + "hive.ql.io.orc.ColumnStatistics\022\026\n\016rowIn", + "dexStride\030\010 \001(\r\"\305\001\n\nPostScript\022\024\n\014footer" + + "Length\030\001 \001(\004\022F\n\013compression\030\002 \001(\01621.org." + + "apache.hadoop.hive.ql.io.orc.Compression" + + "Kind\022\034\n\024compressionBlockSize\030\003 \001(\004\022\023\n\007ve" + + "rsion\030\004 \003(\rB\002\020\001\022\026\n\016metadataLength\030\005 \001(\004\022" + + "\016\n\005magic\030\300> \001(\t*:\n\017CompressionKind\022\010\n\004NO" + + "NE\020\000\022\010\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -15296,18 +16740,30 @@ public Builder setMagicBytes( com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_UserMetadataItem_descriptor, new java.lang.String[] { "Name", "Value", }); - internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor = + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor = getDescriptor().getMessageTypes().get(16); + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_StripeStatistics_descriptor, + new java.lang.String[] { "ColStats", }); + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor = + getDescriptor().getMessageTypes().get(17); + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_hadoop_hive_ql_io_orc_Metadata_descriptor, + new java.lang.String[] { "StripeStats", }); + internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor = + getDescriptor().getMessageTypes().get(18); internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_Footer_descriptor, new java.lang.String[] { "HeaderLength", "ContentLength", "Stripes", "Types", "Metadata", "NumberOfRows", "Statistics", "RowIndexStride", }); internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor = - getDescriptor().getMessageTypes().get(17); + getDescriptor().getMessageTypes().get(19); internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_org_apache_hadoop_hive_ql_io_orc_PostScript_descriptor, - new java.lang.String[] { "FooterLength", "Compression", "CompressionBlockSize", "Version", "Magic", }); + new java.lang.String[] { "FooterLength", "Compression", "CompressionBlockSize", "Version", "MetadataLength", "Magic", }); return null; } }; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java index d2d9912..cd98a52 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java @@ -41,8 +41,17 @@ public static void main(String[] args) throws Exception { System.out.println("Compression size: " + reader.getCompressionSize()); } System.out.println("Type: " + reader.getObjectInspector().getTypeName()); + System.out.println("\nStripe Statistics:"); + Metadata metadata = reader.getMetadata(); + for (int n = 0; n < metadata.getStripeStatisticsList().size(); n++) { + System.out.println(" Stripe " + (n + 1) + ":"); + StripeStatistics ss = metadata.getStripeStatisticsList().get(n); + for (int i = 0; i < ss.getColumnStatistics().length; ++i) { + System.out.println(" Column " + i + ": " + ss.getColumnStatistics()[i].toString()); + } + } ColumnStatistics[] stats = reader.getStatistics(); - System.out.println("\nStatistics:"); + System.out.println("\nFile Statistics:"); for(int i=0; i < stats.length; ++i) { System.out.println(" Column " + i + ": " + stats[i].toString()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/Metadata.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/Metadata.java new file mode 100644 index 0000000..e2ff621 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/Metadata.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.util.Iterator; +import java.util.List; + +import com.google.common.collect.Lists; + +public class Metadata { + + private final OrcProto.Metadata metadata; + + public Metadata(OrcProto.Metadata m) { + this.metadata = m; + } + + /** + * Return iterable stripe level column statistics + * + * @return + */ + public Iterable getStripeStatistics() { + return new Iterable() { + private final Iterator inner = + metadata.getStripeStatsList().iterator(); + + @Override + public Iterator iterator() { + return new Iterator() { + + @Override + public boolean hasNext() { + return inner.hasNext(); + } + + @Override + public StripeStatistics next() { + return new StripeStatistics(inner.next().getColStatsList()); + } + + @Override + public void remove() { + inner.remove(); + } + + }; + } + }; + } + + /** + * Return list of stripe level column statistics + * + * @return list of stripe statistics + */ + public List getStripeStatisticsList() { + List result = Lists.newArrayList(); + for (OrcProto.StripeStatistics ss : metadata.getStripeStatsList()) { + result.add(new StripeStatistics(ss.getColStatsList())); + } + return result; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java index 37d7d86..6a0a488 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java @@ -18,13 +18,13 @@ package org.apache.hadoop.hive.ql.io.orc; -import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + /** * The interface for reading ORC files. * @@ -55,14 +55,14 @@ * Get the user metadata keys. * @return the set of metadata keys */ - Iterable getMetadataKeys(); + Iterable getUserMetadataKeys(); /** * Get a user metadata value. * @param key a key given by the user * @return the bytes associated with the given key */ - ByteBuffer getMetadataValue(String key); + ByteBuffer getUserMetadataValue(String key); /** * Get the compression kind. @@ -108,6 +108,12 @@ ColumnStatistics[] getStatistics(); /** + * Get the metadata information like stripe level column statistics etc. + * @return the information about the column + */ + Metadata getMetadata(); + + /** * Get the list of types contained in the file. The root type is the first * type in the list. * @return the list of flattened types @@ -134,6 +140,7 @@ * @throws IOException * @deprecated */ + @Deprecated RecordReader rows(long offset, long length, boolean[] include) throws IOException; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index 2f3d735..2dc6456 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -53,6 +53,7 @@ private final CompressionKind compressionKind; private final CompressionCodec codec; private final int bufferSize; + private final OrcProto.Metadata metadata; private final OrcProto.Footer footer; private final ObjectInspector inspector; private long deserializedSize = -1; @@ -109,7 +110,7 @@ public long getNumberOfRows() { } @Override - public Iterable getMetadataKeys() { + public Iterable getUserMetadataKeys() { List result = new ArrayList(); for(OrcProto.UserMetadataItem item: footer.getMetadataList()) { result.add(item.getName()); @@ -118,7 +119,7 @@ public long getNumberOfRows() { } @Override - public ByteBuffer getMetadataValue(String key) { + public ByteBuffer getUserMetadataValue(String key) { for(OrcProto.UserMetadataItem item: footer.getMetadataList()) { if (item.hasName() && item.getName().equals(key)) { return item.getValue().asReadOnlyByteBuffer(); @@ -290,6 +291,7 @@ static void checkOrcVersion(Log log, Path path, List version) { OrcProto.PostScript ps = OrcProto.PostScript.parseFrom(in); checkOrcVersion(LOG, path, ps.getVersionList()); int footerSize = (int) ps.getFooterLength(); + int metadataSize = (int) ps.getMetadataLength(); bufferSize = (int) ps.getCompressionBlockSize(); switch (ps.getCompression()) { case NONE: @@ -308,7 +310,7 @@ static void checkOrcVersion(Log log, Path path, List version) { throw new IllegalArgumentException("Unknown compression"); } codec = WriterImpl.createCodec(compressionKind); - int extra = Math.max(0, psLen + 1 + footerSize - readSize); + int extra = Math.max(0, psLen + 1 + footerSize + metadataSize - readSize); if (extra > 0) { file.seek(size - readSize - extra); ByteBuffer extraBuf = ByteBuffer.allocate(extra + readSize); @@ -317,16 +319,29 @@ static void checkOrcVersion(Log log, Path path, List version) { extraBuf.position(extra); extraBuf.put(buffer); buffer = extraBuf; - buffer.position(0); - buffer.limit(footerSize); + buffer.position(metadataSize); + buffer.limit(footerSize + metadataSize); } else { buffer.position(psOffset - footerSize); buffer.limit(psOffset); } + // read footer InputStream instream = InStream.create("footer", new ByteBuffer[]{buffer}, new long[]{0L}, footerSize, codec, bufferSize); footer = OrcProto.Footer.parseFrom(instream); inspector = OrcStruct.createObjectInspector(0, footer.getTypesList()); + + // read metadata + if(extra > 0) { + buffer.position(0); + buffer.limit(metadataSize); + } else { + buffer.position(psOffset - (footerSize + metadataSize)); + buffer.limit(psOffset - footerSize); + } + instream = InStream.create("metadata", new ByteBuffer[]{buffer}, + new long[]{0L}, metadataSize, codec, bufferSize); + metadata = OrcProto.Metadata.parseFrom(instream); file.close(); } @@ -470,4 +485,10 @@ private int getLastIdx() { return Collections.max(indices); } + @Override + public Metadata getMetadata() { + return new Metadata(metadata); + } + + } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java new file mode 100644 index 0000000..0df0ea8 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeStatistics.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.util.List; + +public class StripeStatistics { + private final List cs; + + public StripeStatistics(List list) { + this.cs = list; + } + + /** + * Return list of column statistics + * + * @return column stats + */ + public ColumnStatistics[] getColumnStatistics() { + ColumnStatistics[] result = new ColumnStatistics[cs.size()]; + for (int i = 0; i < result.length; ++i) { + result[i] = ColumnStatisticsImpl.deserialize(cs.get(i)); + } + return result; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java index e4ade90..6515f72 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry; +import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics; import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type; import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.hive.serde2.io.DateWritable; @@ -63,6 +64,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; +import com.google.common.collect.Lists; import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; @@ -124,7 +126,6 @@ private final boolean buildIndex; private final MemoryManager memoryManager; private final OrcFile.Version version; - private final Configuration conf; WriterImpl(FileSystem fs, @@ -410,6 +411,7 @@ public Configuration getConfiguration() { private final BitFieldWriter isPresent; private final boolean isCompressed; protected final ColumnStatisticsImpl indexStatistics; + protected final ColumnStatisticsImpl stripeColStatistics; private final ColumnStatisticsImpl fileStatistics; protected TreeWriter[] childrenWriters; protected final RowIndexPositionRecorder rowIndexPosition; @@ -418,7 +420,7 @@ public Configuration getConfiguration() { private final PositionedOutputStream rowIndexStream; private boolean foundNulls; private OutStream isPresentOutStream; - protected final boolean useDirectV2Encoding; + private final List stripeStatsBuilders; /** * Create a tree writer. @@ -434,7 +436,6 @@ public Configuration getConfiguration() { this.isCompressed = streamFactory.isCompressed(); this.id = columnId; this.inspector = inspector; - this.useDirectV2Encoding = true; if (nullable) { isPresentOutStream = streamFactory.createStream(id, OrcProto.Stream.Kind.PRESENT); @@ -444,11 +445,13 @@ public Configuration getConfiguration() { } this.foundNulls = false; indexStatistics = ColumnStatisticsImpl.create(inspector); + stripeColStatistics = ColumnStatisticsImpl.create(inspector); fileStatistics = ColumnStatisticsImpl.create(inspector); childrenWriters = new TreeWriter[0]; rowIndex = OrcProto.RowIndex.newBuilder(); rowIndexEntry = OrcProto.RowIndexEntry.newBuilder(); rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry); + stripeStatsBuilders = Lists.newArrayList(); if (streamFactory.buildIndex()) { rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Kind.ROW_INDEX); @@ -461,6 +464,10 @@ public Configuration getConfiguration() { return rowIndex; } + protected ColumnStatisticsImpl getStripeStatistics() { + return stripeColStatistics; + } + protected ColumnStatisticsImpl getFileStatistics() { return fileStatistics; } @@ -536,6 +543,12 @@ void writeStripe(OrcProto.StripeFooter.Builder builder, } } + // merge stripe-level column statistics to file statistics and write it to + // stripe statistics + OrcProto.StripeStatistics.Builder stripeStatsBuilder = OrcProto.StripeStatistics.newBuilder(); + writeStripeStatistics(stripeStatsBuilder, this); + stripeStatsBuilders.add(stripeStatsBuilder); + // reset the flag for next stripe foundNulls = false; @@ -553,6 +566,16 @@ void writeStripe(OrcProto.StripeFooter.Builder builder, rowIndexEntry.clear(); } + private void writeStripeStatistics(OrcProto.StripeStatistics.Builder builder, + TreeWriter treeWriter) { + treeWriter.fileStatistics.merge(treeWriter.stripeColStatistics); + builder.addColStats(treeWriter.stripeColStatistics.serialize().build()); + treeWriter.stripeColStatistics.reset(); + for (TreeWriter child : treeWriter.getChildrenWriters()) { + writeStripeStatistics(builder, child); + } + } + TreeWriter[] getChildrenWriters() { return childrenWriters; } @@ -574,7 +597,7 @@ void writeStripe(OrcProto.StripeFooter.Builder builder, * @throws IOException */ void createRowIndexEntry() throws IOException { - fileStatistics.merge(indexStatistics); + stripeColStatistics.merge(indexStatistics); rowIndexEntry.setStatistics(indexStatistics.serialize()); indexStatistics.reset(); rowIndex.addEntry(rowIndexEntry); @@ -1012,7 +1035,7 @@ public void visit(StringRedBlackTree.VisitorContext context */ @Override void createRowIndexEntry() throws IOException { - getFileStatistics().merge(indexStatistics); + getStripeStatistics().merge(indexStatistics); OrcProto.RowIndexEntry.Builder rowIndexEntry = getRowIndexEntry(); rowIndexEntry.setStatistics(indexStatistics.serialize()); indexStatistics.reset(); @@ -1844,6 +1867,21 @@ private void writeFileStatistics(OrcProto.Footer.Builder builder, } } + private int writeMetadata(long bodyLength) throws IOException { + ensureWriter(); + OrcProto.Metadata.Builder builder = OrcProto.Metadata.newBuilder(); + for(OrcProto.StripeStatistics.Builder ssb : treeWriter.stripeStatsBuilders) { + builder.addStripeStats(ssb.build()); + } + + long startPosn = rawWriter.getPos(); + OrcProto.Metadata metadata = builder.build(); + metadata.writeTo(protobufWriter); + protobufWriter.flush(); + writer.flush(); + return (int) (rawWriter.getPos() - startPosn); + } + private int writeFooter(long bodyLength) throws IOException { ensureWriter(); OrcProto.Footer.Builder builder = OrcProto.Footer.newBuilder(); @@ -1874,11 +1912,12 @@ private int writeFooter(long bodyLength) throws IOException { return (int) (rawWriter.getPos() - startPosn); } - private int writePostScript(int footerLength) throws IOException { + private int writePostScript(int footerLength, int metadataLength) throws IOException { OrcProto.PostScript.Builder builder = OrcProto.PostScript.newBuilder() .setCompression(writeCompressionKind(compress)) .setFooterLength(footerLength) + .setMetadataLength(metadataLength) .setMagic(OrcFile.MAGIC) .addVersion(version.getMajor()) .addVersion(version.getMinor()); @@ -1933,8 +1972,9 @@ public void close() throws IOException { // actually close the file synchronized (this) { flushStripe(); - int footerLength = writeFooter(rawWriter.getPos()); - rawWriter.writeByte(writePostScript(footerLength)); + int metadataLength = writeMetadata(rawWriter.getPos()); + int footerLength = writeFooter(rawWriter.getPos() - metadataLength); + rawWriter.writeByte(writePostScript(footerLength, metadataLength)); rawWriter.close(); } } diff --git ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto index 53b93a0..d2a787e 100644 --- ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto +++ ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto @@ -150,6 +150,14 @@ message UserMetadataItem { required bytes value = 2; } +message StripeStatistics { + repeated ColumnStatistics colStats = 1; +} + +message Metadata { + repeated StripeStatistics stripeStats = 1; +} + message Footer { optional uint64 headerLength = 1; optional uint64 contentLength = 2; @@ -174,6 +182,7 @@ message PostScript { optional CompressionKind compression = 2; optional uint64 compressionBlockSize = 3; repeated uint32 version = 4 [packed = true]; + optional uint64 metadataLength = 5; // Leave this last in the record optional string magic = 8000; } diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java index 1dfcb56..1f5f96a 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java @@ -541,6 +541,70 @@ public void testStringAndBinaryStatistics() throws Exception { rows.close(); } + + @Test + public void testStripeLevelStats() throws Exception { + ObjectInspector inspector; + synchronized (TestOrcFile.class) { + inspector = ObjectInspectorFactory.getReflectionObjectInspector + (InnerStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + } + Writer writer = OrcFile.createWriter(testFilePath, + OrcFile.writerOptions(conf) + .inspector(inspector) + .stripeSize(100000) + .bufferSize(10000)); + for (int i = 0; i < 11000; i++) { + if (i >= 5000) { + if (i >= 10000) { + writer.addRow(new InnerStruct(3, "three")); + } else { + writer.addRow(new InnerStruct(2, "two")); + } + } else { + writer.addRow(new InnerStruct(1, "one")); + } + } + + writer.close(); + Reader reader = OrcFile.createReader(fs, testFilePath); + Metadata metadata = reader.getMetadata(); + int numStripes = metadata.getStripeStatisticsList().size(); + assertEquals(3, numStripes); + StripeStatistics ss1 = metadata.getStripeStatisticsList().get(0); + StripeStatistics ss2 = metadata.getStripeStatisticsList().get(1); + StripeStatistics ss3 = metadata.getStripeStatisticsList().get(2); + assertEquals(4996, ss1.getColumnStatistics()[0].getNumberOfValues()); + assertEquals(5000, ss2.getColumnStatistics()[0].getNumberOfValues()); + assertEquals(1004, ss3.getColumnStatistics()[0].getNumberOfValues()); + + assertEquals(4996, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getNumberOfValues()); + assertEquals(5000, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getNumberOfValues()); + assertEquals(1004, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getNumberOfValues()); + assertEquals(1, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getMinimum()); + assertEquals(1, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getMinimum()); + assertEquals(2, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getMinimum()); + assertEquals(1, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getMaximum()); + assertEquals(2, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getMaximum()); + assertEquals(3, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getMaximum()); + assertEquals(4996, ((IntegerColumnStatistics)ss1.getColumnStatistics()[1]).getSum()); + assertEquals(9996, ((IntegerColumnStatistics)ss2.getColumnStatistics()[1]).getSum()); + assertEquals(3008, ((IntegerColumnStatistics)ss3.getColumnStatistics()[1]).getSum()); + + assertEquals(4996, ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getNumberOfValues()); + assertEquals(5000, ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getNumberOfValues()); + assertEquals(1004, ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getNumberOfValues()); + assertEquals("one", ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getMinimum()); + assertEquals("one", ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getMinimum()); + assertEquals("three", ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getMinimum()); + assertEquals("one", ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getMaximum()); + assertEquals("two", ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getMaximum()); + assertEquals("two", ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getMaximum()); + assertEquals(14988, ((StringColumnStatistics)ss1.getColumnStatistics()[2]).getSum()); + assertEquals(15000, ((StringColumnStatistics)ss2.getColumnStatistics()[2]).getSum()); + assertEquals(5012, ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getSum()); + } + @Test public void test1() throws Exception { ObjectInspector inspector; @@ -549,12 +613,12 @@ public void test1() throws Exception { (BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } Writer writer = OrcFile.createWriter(testFilePath, - OrcFile.writerOptions(conf) - .inspector(inspector) - .stripeSize(100000) - .bufferSize(10000)); + OrcFile.writerOptions(conf) + .inspector(inspector) + .stripeSize(100000) + .bufferSize(10000)); writer.addRow(new BigRow(false, (byte) 1, (short) 1024, 65536, - Long.MAX_VALUE, (float) 1.0, -15.0, bytes(0,1,2,3,4), "hi", + Long.MAX_VALUE, (float) 1.0, -15.0, bytes(0, 1, 2, 3, 4), "hi", new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), list(inner(3, "good"), inner(4, "bad")), map())); @@ -562,10 +626,12 @@ public void test1() throws Exception { Long.MAX_VALUE, (float) 2.0, -5.0, bytes(), "bye", new MiddleStruct(inner(1, "bye"), inner(2, "sigh")), list(inner(100000000, "cat"), inner(-100000, "in"), inner(1234, "hat")), - map(inner(5,"chani"), inner(1,"mauddib")))); + map(inner(5, "chani"), inner(1, "mauddib")))); writer.close(); Reader reader = OrcFile.createReader(fs, testFilePath); + Metadata metadata = reader.getMetadata(); + // check the stats ColumnStatistics[] stats = reader.getStatistics(); assertEquals(2, stats[1].getNumberOfValues()); @@ -580,14 +646,12 @@ public void test1() throws Exception { assertEquals("count: 2 min: 1024 max: 2048 sum: 3072", stats[3].toString()); - assertEquals(Long.MAX_VALUE, - ((IntegerColumnStatistics) stats[5]).getMaximum()); - assertEquals(Long.MAX_VALUE, - ((IntegerColumnStatistics) stats[5]).getMinimum()); - assertEquals(false, ((IntegerColumnStatistics) stats[5]).isSumDefined()); - assertEquals("count: 2 min: 9223372036854775807 max: 9223372036854775807", - stats[5].toString()); - + StripeStatistics ss = metadata.getStripeStatisticsList().get(0); + assertEquals(2, ss.getColumnStatistics()[0].getNumberOfValues()); + assertEquals(1, ((BooleanColumnStatistics) ss.getColumnStatistics()[1]).getTrueCount()); + assertEquals(1024, ((IntegerColumnStatistics) ss.getColumnStatistics()[3]).getMinimum()); + assertEquals(2048, ((IntegerColumnStatistics) ss.getColumnStatistics()[3]).getMaximum()); + assertEquals(3072, ((IntegerColumnStatistics) ss.getColumnStatistics()[3]).getSum()); assertEquals(-15.0, ((DoubleColumnStatistics) stats[7]).getMinimum()); assertEquals(-5.0, ((DoubleColumnStatistics) stats[7]).getMaximum()); assertEquals(-20.0, ((DoubleColumnStatistics) stats[7]).getSum(), 0.00001); @@ -883,7 +947,7 @@ public void emptyFile() throws Exception { assertEquals(CompressionKind.NONE, reader.getCompression()); assertEquals(0, reader.getNumberOfRows()); assertEquals(0, reader.getCompressionSize()); - assertEquals(false, reader.getMetadataKeys().iterator().hasNext()); + assertEquals(false, reader.getUserMetadataKeys().iterator().hasNext()); assertEquals(3, reader.getContentLength()); assertEquals(false, reader.getStripes().iterator().hasNext()); } @@ -916,18 +980,18 @@ public void metaData() throws Exception { writer.addUserMetadata("clobber", byteBuf(5,7,11,13,17,19)); writer.close(); Reader reader = OrcFile.createReader(fs, testFilePath); - assertEquals(byteBuf(5,7,11,13,17,19), reader.getMetadataValue("clobber")); + assertEquals(byteBuf(5,7,11,13,17,19), reader.getUserMetadataValue("clobber")); assertEquals(byteBuf(1,2,3,4,5,6,7,-1,-2,127,-128), - reader.getMetadataValue("my.meta")); - assertEquals(bigBuf, reader.getMetadataValue("big")); + reader.getUserMetadataValue("my.meta")); + assertEquals(bigBuf, reader.getUserMetadataValue("big")); try { - reader.getMetadataValue("unknown"); + reader.getUserMetadataValue("unknown"); assertTrue(false); } catch (IllegalArgumentException iae) { // PASS } int i = 0; - for(String key: reader.getMetadataKeys()) { + for(String key: reader.getUserMetadataKeys()) { if ("my.meta".equals(key) || "clobber".equals(key) || "big".equals(key)) { @@ -1031,7 +1095,7 @@ public void testUnionAndTimestamp() throws Exception { writer.addRow(row); writer.close(); Reader reader = OrcFile.createReader(fs, testFilePath); - assertEquals(false, reader.getMetadataKeys().iterator().hasNext()); + assertEquals(false, reader.getUserMetadataKeys().iterator().hasNext()); assertEquals(5309, reader.getNumberOfRows()); DecimalColumnStatistics stats = (DecimalColumnStatistics) reader.getStatistics()[5]; diff --git ql/src/test/resources/orc-file-dump-dictionary-threshold.out ql/src/test/resources/orc-file-dump-dictionary-threshold.out index bac7465..8687b26 100644 --- ql/src/test/resources/orc-file-dump-dictionary-threshold.out +++ ql/src/test/resources/orc-file-dump-dictionary-threshold.out @@ -4,7 +4,34 @@ Compression: ZLIB Compression size: 10000 Type: struct -Statistics: +Stripe Statistics: + Stripe 1: + Column 0: count: 4000 + Column 1: count: 4000 min: -2147115959 max: 2145911404 sum: 71315665983 + Column 2: count: 4000 min: -9211329013123260308 max: 9217851628057711416 + Column 3: count: 4000 min: Darkness,-230 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788 sum: 245096 + Stripe 2: + Column 0: count: 5000 + Column 1: count: 5000 min: -2147390285 max: 2146838901 sum: 107869424275 + Column 2: count: 5000 min: -9222178666167296739 max: 9221301751385928177 + Column 3: count: 5000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984 sum: 972748 + Stripe 3: + Column 0: count: 5000 + Column 1: count: 5000 min: -2145928262 max: 2147224606 sum: 38276585043 + Column 2: count: 5000 min: -9221963099397084326 max: 9222722740629726770 + Column 3: count: 5000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766 sum: 1753024 + Stripe 4: + Column 0: count: 5000 + Column 1: count: 5000 min: -2145378214 max: 2147453086 sum: -43469576640 + Column 2: count: 5000 min: -9222731174895935707 max: 9222919052987871506 + Column 3: count: 5000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788 sum: 2636664 + Stripe 5: + Column 0: count: 2000 + Column 1: count: 2000 min: -2143595397 max: 2144595861 sum: -64863580335 + Column 2: count: 2000 min: -9212379634781416464 max: 9208134757538374043 + Column 3: count: 2000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214-18444-18446-18724-18912-18952-19164 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 sum: 1302706 + +File Statistics: Column 0: count: 21000 Column 1: count: 21000 min: -2147390285 max: 2147453086 sum: 109128518326 Column 2: count: 21000 min: -9222731174895935707 max: 9222919052987871506 diff --git ql/src/test/resources/orc-file-dump.out ql/src/test/resources/orc-file-dump.out index 5b5eb2c..7a5b907 100644 --- ql/src/test/resources/orc-file-dump.out +++ ql/src/test/resources/orc-file-dump.out @@ -4,7 +4,34 @@ Compression: ZLIB Compression size: 10000 Type: struct -Statistics: +Stripe Statistics: + Stripe 1: + Column 0: count: 5000 + Column 1: count: 5000 min: -2146021688 max: 2147223299 sum: 515792826 + Column 2: count: 5000 min: -9218592812243954469 max: 9221614132680747961 + Column 3: count: 5000 min: Darkness, max: worst sum: 19280 + Stripe 2: + Column 0: count: 5000 + Column 1: count: 5000 min: -2146733128 max: 2147001622 sum: 7673427 + Column 2: count: 5000 min: -9220818777591257749 max: 9222259462014003839 + Column 3: count: 5000 min: Darkness, max: worst sum: 19504 + Stripe 3: + Column 0: count: 5000 + Column 1: count: 5000 min: -2146993718 max: 2147378179 sum: 132660742551 + Column 2: count: 5000 min: -9218342074710552826 max: 9222303228623055266 + Column 3: count: 5000 min: Darkness, max: worst sum: 19641 + Stripe 4: + Column 0: count: 5000 + Column 1: count: 5000 min: -2146658006 max: 2145520931 sum: 8533549236 + Column 2: count: 5000 min: -9222758097219661129 max: 9221043130193737406 + Column 3: count: 5000 min: Darkness, max: worst sum: 19470 + Stripe 5: + Column 0: count: 1000 + Column 1: count: 1000 min: -2146245500 max: 2146378640 sum: 51299706363 + Column 2: count: 1000 min: -9208193203370316142 max: 9218567213558056476 + Column 3: count: 1000 min: Darkness, max: worst sum: 3866 + +File Statistics: Column 0: count: 21000 Column 1: count: 21000 min: -2146993718 max: 2147378179 sum: 193017464403 Column 2: count: 21000 min: -9222758097219661129 max: 9222303228623055266