diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index 4033b379de..2947c167fc 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -74,8 +74,6 @@ import org.apache.hadoop.hive.llap.io.metadata.MetadataCache; import org.apache.hadoop.hive.llap.io.metadata.MetadataCache.LlapBufferOrBuffers; import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata; -import org.apache.hadoop.hive.ql.exec.DDLTask; -import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HdfsUtils; import org.apache.orc.CompressionKind; import org.apache.orc.DataReader; @@ -766,7 +764,8 @@ private boolean determineRgsToRead(int rowIndexStride, if (sarg != null && rowIndexStride != 0) { sargApp = new RecordReaderImpl.SargApplier(sarg, rowIndexStride, evolution, - OrcFile.WriterVersion.from(fileMetadata.getWriterVersionNum())); + OrcFile.WriterVersion.from(OrcFile.WriterImplementation.ORC_JAVA, fileMetadata.getWriterVersionNum()), + false); } boolean hasAnyData = false; // stripeRgs should have been initialized by this time with an empty array. diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java index 1cfe92978a..ba2575f8a4 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java @@ -563,6 +563,23 @@ public void setCurrentStripeOffsets(long currentKnownTornStart, public CompressionCodec getCompressionCodec() { return null; } + + @Override + public long getFileBytes(int column) { + long size = 0L; + List l = this.colStreams.get(column); + if (l == null) { + return size; + } + for (CacheOutputReceiver c : l) { + if (!c.suppressed && c.getName().getArea() != StreamName.Area.INDEX) { + for (MemoryBuffer buffer : c.getData()) { + size += buffer.getByteBufferRaw().limit(); + } + } + } + return size; + } } private interface CacheOutput { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java index bf139c071c..89ad4aa8cd 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/ConsumerFileMetadata.java @@ -20,6 +20,7 @@ import java.util.List; import org.apache.orc.CompressionKind; +import org.apache.orc.FileFormatException; import org.apache.orc.OrcProto.Type; import org.apache.orc.TypeDescription; @@ -27,5 +28,5 @@ int getStripeCount(); CompressionKind getCompressionKind(); List getTypes(); - TypeDescription getSchema(); + TypeDescription getSchema() throws FileFormatException; } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java index 0012afb3ef..5cd6f9fa2c 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java @@ -22,7 +22,9 @@ import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.orc.CompressionKind; +import org.apache.orc.FileFormatException; import org.apache.orc.FileMetadata; +import org.apache.orc.OrcFile; import org.apache.orc.OrcProto; import org.apache.orc.OrcProto.StripeStatistics; import org.apache.orc.OrcUtils; @@ -123,6 +125,11 @@ public int getMetadataSize() { return metadataSize; } + @Override + public int getWriterImplementation() { + return OrcFile.WriterImplementation.ORC_JAVA.getId(); + } + @Override public int getWriterVersionNum() { return writerVersionNum; @@ -153,7 +160,7 @@ public int getStripeCount() { return stripes.size(); } - public TypeDescription getSchema() { + public TypeDescription getSchema() throws FileFormatException { return OrcUtils.convertTypeFromProtobuf(this.types, 0); } } diff --git a/pom.xml b/pom.xml index e11339982d..a38673a27d 100644 --- a/pom.xml +++ b/pom.xml @@ -184,7 +184,7 @@ 0.9.3 2.10.0 2.3 - 1.4.3 + 1.5.0-SNAPSHOT 1.10.19 2.0.0-M5 4.1.17.Final diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 2337a350e6..a773ad4e46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -29,7 +29,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.NavigableMap; @@ -115,6 +114,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hive.common.util.Ref; import org.apache.orc.ColumnStatistics; +import org.apache.orc.FileFormatException; import org.apache.orc.OrcProto; import org.apache.orc.OrcProto.Footer; import org.apache.orc.OrcUtils; @@ -2146,7 +2146,7 @@ static Reader createOrcReaderForSplit(Configuration conf, OrcSplit orcSplit) thr public static boolean[] pickStripesViaTranslatedSarg(SearchArgument sarg, OrcFile.WriterVersion writerVersion, List types, - List stripeStats, int stripeCount) { + List stripeStats, int stripeCount) throws FileFormatException { LOG.info("Translated ORC pushdown predicate: " + sarg); assert sarg != null; if (stripeStats == null || writerVersion == OrcFile.WriterVersion.ORIGINAL) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java index 42532f9a0e..ff47a5154a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java @@ -23,14 +23,12 @@ import java.util.Arrays; import java.util.List; -import org.apache.curator.shaded.com.google.common.base.Preconditions; import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch; import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch; import org.apache.orc.CompressionCodec; import org.apache.orc.TypeDescription; -import org.apache.orc.TypeDescription.Category; import org.apache.orc.impl.PositionProvider; import org.apache.orc.impl.SettableUncompressedStream; import org.apache.orc.impl.TreeReaderFactory; @@ -1051,7 +1049,8 @@ private DecimalStreamReader(int columnId, int precision, int scale, boolean isFileCompressed, OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, List vectors) throws IOException { - super(columnId, presentStream, valueStream, scaleStream, encoding, context); + super(columnId, presentStream, valueStream, scaleStream, encoding, + precision, scale, context); this._isFileCompressed = isFileCompressed; this._presentStream = presentStream; this._valueStream = valueStream;