diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 409fc90099..b6e4060dba 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -184,6 +184,8 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ join_nullsafe.q,\ leftsemijoin.q,\ limit_pushdown.q,\ + llap_acid.q,\ + llap_acid_fast.q,\ load_dyn_part1.q,\ load_dyn_part2.q,\ load_dyn_part3.q,\ diff --git llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java index 22ca025e90..cd64087842 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java @@ -138,7 +138,7 @@ InputSplit split, JobConf job, Reporter reporter) throws IOException { boolean useLlapIo = true; if (split instanceof LlapAwareSplit) { - useLlapIo = ((LlapAwareSplit) split).canUseLlapIo(); + useLlapIo = ((LlapAwareSplit) split).canUseLlapIo(job); } if (useLlapIo) return null; diff --git llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java index 53c9bae5c1..601f1bdf63 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java @@ -229,4 +229,8 @@ public void close() { } executor.shutdownNow(); } + + public ColumnVectorProducer getOrcCvp() { + return orcCvp; + } } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java index bbfe856843..dcb2fe9244 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java @@ -103,6 +103,11 @@ public static LlapRecordReader create(JobConf job, FileSplit split, List unused0, Deserializer unused1, Reporter reporter, JobConf job, Map unused2) throws IOException { cacheMetrics.incrCacheReadRequests(); - OrcEncodedDataConsumer edc = new OrcEncodedDataConsumer(consumer, columnIds.size(), - _skipCorrupt, counters, ioMetrics); - OrcEncodedDataReader reader = new OrcEncodedDataReader( - lowLevelCache, bufferManager, metadataCache, conf, job, split, columnIds, sarg, - columnNames, edc, counters, readerSchema); + + final boolean isAcidRead = HiveConf.getBoolVar(job, + HiveConf.ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN); + + final boolean isOriginal; + final boolean hasBase; + final boolean hasDelta; + + if (split instanceof OrcSplit) { + final OrcSplit orcSplit = (OrcSplit) split; + isOriginal = orcSplit.isOriginal(); + hasBase = orcSplit.hasBase(); + hasDelta = orcSplit.getDeltas() != null && !orcSplit.getDeltas().isEmpty(); + } else { + isOriginal = true; + hasBase = false; + hasDelta = false; + } + + final AcidUtils.AcidOperationalProperties acidOperationalProperties + = AcidUtils.getAcidOperationalProperties(job); + final boolean isSplitUpdate = acidOperationalProperties.isSplitUpdate(); + OrcEncodedDataConsumer edc = null; + OrcEncodedDataReader reader = null; + + if (isOriginal) { + if (!hasDelta) { + LOG.debug("Original only"); + edc = new OrcEncodedDataConsumer(consumer, columnIds.size(), _skipCorrupt, counters, + ioMetrics); + reader = new OrcEncodedDataReader(lowLevelCache, bufferManager, metadataCache, conf, job, + split, columnIds, sarg, columnNames, edc, counters, readerSchema); + } + } else { + if (isAcidRead && hasBase) { + if (hasDelta) { + if (isSplitUpdate) { + LOG.debug("Base with delete deltas"); + consumer = new RowProjectionConsumer(consumer, columnIds); + consumer = new RecordReaderConsumer(consumer, readerSchema.getChildren().size()) { + @Override + RecordReader getMergedReader( + RecordReader baseReader) throws IOException { + return new VectorizedOrcAcidRowBatchReader(split, job, reporter, baseReader); + } + }; + edc = new OrcEncodedDataConsumer(consumer, OrcRecordUpdater.FIELDS, _skipCorrupt, + counters, ioMetrics); + reader = new OrcEncodedDataReader(lowLevelCache, bufferManager, metadataCache, conf, + job, split, columnIds, sarg, columnNames, edc, counters, readerSchema); + } + } else { + LOG.debug("Base only"); + consumer = new RowProjectionConsumer(consumer, columnIds); + edc = new OrcEncodedDataConsumer(consumer, OrcRecordUpdater.FIELDS, _skipCorrupt, + counters, ioMetrics); + reader = new OrcEncodedDataReader(lowLevelCache, bufferManager, metadataCache, conf, job, + split, columnIds, sarg, columnNames, edc, counters, readerSchema); + } + } + } edc.init(reader, reader); return edc; } + + static abstract class RecordReaderConsumer implements Consumer { + protected final Consumer consumer; + protected final RecordReader mergedReader; + protected final RecordReader baseReader; + protected final VectorizedRowBatch processedData; + protected ColumnVectorBatch arrivedData; + + RecordReaderConsumer(Consumer consumer, int numCols) throws IOException { + this.consumer = consumer; + this.processedData = new VectorizedRowBatch(numCols); + this.processedData.setPartitionInfo(numCols, 0); + this.baseReader = + new RecordReader() { + @Override + public boolean next(NullWritable key, VectorizedRowBatch value) throws IOException { + value.size = arrivedData.size; + for (int i = 0; i < OrcRecordUpdater.FIELDS; i++) { + value.cols[i] = arrivedData.cols[i]; + } + return true; + } + + @Override + public NullWritable createKey() { + return NullWritable.get(); + } + + @Override + public VectorizedRowBatch createValue() { + final VectorizedRowBatch batch = new VectorizedRowBatch(OrcRecordUpdater.FIELDS); + batch.setPartitionInfo(OrcRecordUpdater.FIELDS, 0); + return batch; + } + + @Override + public long getPos() throws IOException { + return 0; + } + + @Override + public void close() throws IOException { + setDone(); + } + + @Override + public float getProgress() throws IOException { + return 0; + } + }; + this.mergedReader = getMergedReader(baseReader); + } + + abstract RecordReader getMergedReader( + RecordReader baseReader) throws IOException; + + @Override + public void setDone() { + consumer.setDone(); + } + + @Override + public void setError(Throwable t) { + consumer.setError(t); + } + + @Override + public void consumeData(ColumnVectorBatch data) { + try { + this.arrivedData = data; + mergedReader.next(null, processedData); + // VectorizedRowBatch to ColumnVectorBatch + for (int i = 0; i < processedData.projectionSize; i++) { + data.swapColumnVector(i, processedData.cols, processedData.projectedColumns[i]); + } + data.size = processedData.size; + int j = 0; + if (processedData.selectedInUse) { + for (int i = 0; i < processedData.size; i++) { + for (ColumnVector columnVector : data.cols) { + columnVector.setElement(j++, processedData.selected[i], columnVector); + } + } + } + consumer.consumeData(data); + } catch (IOException e) { + consumer.setError(e); + throw new RuntimeException(e); + } + } + } + + private class RowProjectionConsumer implements Consumer { + private final Consumer consumer; + private final ColumnVectorBatch output; + private final List columnIds; + + public RowProjectionConsumer(Consumer consumer, List columnIds) { + this.consumer = consumer; + this.output = new ColumnVectorBatch(columnIds.size()); + this.columnIds = columnIds; + } + + @Override + public void consumeData(ColumnVectorBatch data) { + final StructColumnVector rowColumnVector = (StructColumnVector) data.cols[OrcRecordUpdater.ROW]; + output.size = data.size; + final int size = columnIds.size(); + for (int i = 0; i < size; i++) { + output.swapColumnVector(i, rowColumnVector.fields, columnIds.get(i)); + } + consumer.consumeData(output); + } + + @Override + public void setDone() { + consumer.setDone(); + } + + @Override + public void setError(Throwable t) { + consumer.setError(t); + } + } } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index 2a76f5c4da..e8bb44b54e 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -26,6 +26,8 @@ import java.util.List; import org.apache.hadoop.hive.llap.counters.LlapIOCounters; +import org.apache.hadoop.hive.ql.io.AcidInputFormat; +import org.apache.orc.OrcUtils; import org.apache.orc.TypeDescription; import org.apache.orc.impl.DataReaderProperties; import org.apache.orc.impl.OrcIndex; @@ -203,7 +205,18 @@ public OrcEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager buff if (readerSchema == null) { readerSchema = fileMetadata.getSchema(); } - globalIncludes = OrcInputFormat.genIncludedColumns(readerSchema, includedColumnIds); + final boolean[] readerIncludes = OrcInputFormat.genIncludedColumns(readerSchema, includedColumnIds); + if (split instanceof OrcSplit) { + final OrcSplit orcSplit = (OrcSplit) split; + final List deltas = orcSplit.getDeltas(); + if (orcSplit.isOriginal() && (deltas == null || deltas.isEmpty())) { + globalIncludes = readerIncludes; + } else { + globalIncludes = OrcInputFormat.shiftReaderIncludedForAcid(readerIncludes); + } + } else { + globalIncludes = readerIncludes; + } // Do not allow users to override zero-copy setting. The rest can be taken from user config. boolean useZeroCopy = OrcConf.USE_ZEROCOPY.getBoolean(daemonConf); if (useZeroCopy != OrcConf.USE_ZEROCOPY.getBoolean(jobConf)) { @@ -211,7 +224,7 @@ public OrcEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager buff jobConf.setBoolean(OrcConf.USE_ZEROCOPY.getAttribute(), useZeroCopy); } this.jobConf = jobConf; - Reader.Options options = new Reader.Options(jobConf).include(globalIncludes); + Reader.Options options = new Reader.Options(jobConf).include(readerIncludes); evolution = new SchemaEvolution(fileMetadata.getSchema(), readerSchema, options); consumer.setFileMetadata(fileMetadata); consumer.setIncludedColumns(globalIncludes); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/LlapAwareSplit.java ql/src/java/org/apache/hadoop/hive/ql/io/LlapAwareSplit.java index ead4678f64..20e134541c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/LlapAwareSplit.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/LlapAwareSplit.java @@ -17,11 +17,13 @@ */ package org.apache.hadoop.hive.ql.io; +import org.apache.hadoop.conf.Configuration; + /** * Split that is aware that it could be executed in LLAP. Allows LlapInputFormat to do * a last-minute check to see of LLAP IO pipeline should be used for this particular split. * By default, there is no such check - whatever is sent in is attempted with LLAP IO. */ public interface LlapAwareSplit { - boolean canUseLlapIo(); + boolean canUseLlapIo(Configuration conf); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index f9e17a9fd4..013fdeadf7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -1580,20 +1580,19 @@ private long computeProjectionSize(List fileTypes, } return ReaderImpl.getRawDataSizeFromColIndices(internalColIds, fileTypes, stats); } + } - private boolean[] shiftReaderIncludedForAcid(boolean[] included) { - // We always need the base row - included[0] = true; - boolean[] newIncluded = new boolean[included.length + OrcRecordUpdater.FIELDS]; - Arrays.fill(newIncluded, 0, OrcRecordUpdater.FIELDS, true); - for(int i= 0; i < included.length; ++i) { - newIncluded[i + OrcRecordUpdater.FIELDS] = included[i]; - } - return newIncluded; + public static boolean[] shiftReaderIncludedForAcid(boolean[] included) { + // We always need the base row + included[0] = true; + boolean[] newIncluded = new boolean[included.length + OrcRecordUpdater.FIELDS]; + Arrays.fill(newIncluded, 0, OrcRecordUpdater.FIELDS, true); + for (int i = 0; i < included.length; ++i) { + newIncluded[i + OrcRecordUpdater.FIELDS] = included[i]; } + return newIncluded; } - /** Class intended to update two values from methods... Java-related cruft. */ @VisibleForTesting static final class CombinedCtx { diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java index 65f4a24750..6cf037d520 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java @@ -71,8 +71,8 @@ final static int BUCKET = 2; final static int ROW_ID = 3; final static int CURRENT_TRANSACTION = 4; - final static int ROW = 5; - final static int FIELDS = 6; + public final static int ROW = 5; + public final static int FIELDS = 6; final static int DELTA_BUFFER_SIZE = 16 * 1024; final static long DELTA_STRIPE_SIZE = 16 * 1024 * 1024; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java index d61b24bef3..1bd58d8f55 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java @@ -26,8 +26,11 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.AcidInputFormat; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.ColumnarSplit; import org.apache.hadoop.hive.ql.io.LlapAwareSplit; import org.apache.hadoop.hive.ql.io.SyntheticFileId; @@ -217,8 +220,33 @@ public long getColumnarProjectionSize() { } @Override - public boolean canUseLlapIo() { - return isOriginal && (deltas == null || deltas.isEmpty()); + public boolean canUseLlapIo(Configuration conf) { + final boolean hasDelta = deltas != null && !deltas.isEmpty(); + final boolean isAcidRead = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN); + final AcidUtils.AcidOperationalProperties acidOperationalProperties + = AcidUtils.getAcidOperationalProperties(conf); + final boolean isSplitUpdate = acidOperationalProperties.isSplitUpdate(); + + if (isOriginal) { + if (!hasDelta) { + // Original only + return true; + } + } else { + if (isAcidRead && hasBase) { + if (hasDelta) { + if (isSplitUpdate) { + // Base with delete deltas + return true; + } + } else { + // Base only + return true; + } + } + } + + return false; } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderAdaptor.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderAdaptor.java new file mode 100644 index 0000000000..5cad7b509a --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderAdaptor.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; +import org.apache.hadoop.io.NullWritable; + +import java.io.IOException; + +class RecordReaderAdaptor + implements org.apache.hadoop.mapred.RecordReader { + private final RecordReader innerReader; + private final VectorizedRowBatchCtx rbCtx; + + RecordReaderAdaptor(RecordReader innerReader, VectorizedRowBatchCtx rbCtx) { + this.innerReader = innerReader; + this.rbCtx = rbCtx; + } + + @Override + public boolean next(NullWritable key, VectorizedRowBatch value) throws IOException { + return innerReader.nextBatch(value); + } + + @Override + public NullWritable createKey() { + return NullWritable.get(); + } + + @Override + public VectorizedRowBatch createValue() { + return rbCtx.createVectorizedRowBatch(); + } + + @Override + public long getPos() throws IOException { + return 0; + } + + @Override + public void close() throws IOException { + innerReader.close(); + } + + @Override + public float getProgress() throws IOException { + return innerReader.getProgress(); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java index 29f5a8e375..3d135ecdd2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java @@ -60,7 +60,7 @@ private static final Logger LOG = LoggerFactory.getLogger(VectorizedOrcAcidRowBatchReader.class); - private org.apache.hadoop.hive.ql.io.orc.RecordReader baseReader; + public org.apache.hadoop.mapred.RecordReader baseReader; private VectorizedRowBatchCtx rbCtx; private VectorizedRowBatch vectorizedRowBatchBase; private long offset; @@ -73,6 +73,16 @@ public VectorizedOrcAcidRowBatchReader(InputSplit inputSplit, JobConf conf, Reporter reporter) throws IOException { + this.init(inputSplit, conf, reporter, null); + } + + public VectorizedOrcAcidRowBatchReader(InputSplit inputSplit, JobConf conf, Reporter reporter, + org.apache.hadoop.mapred.RecordReader baseReader) throws IOException { + this.init(inputSplit, conf, reporter, baseReader); + } + + private void init(InputSplit inputSplit, JobConf conf, Reporter reporter, + org.apache.hadoop.mapred.RecordReader baseReader) throws IOException { final boolean isAcidRead = HiveConf.getBoolVar(conf, ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN); final AcidUtils.AcidOperationalProperties acidOperationalProperties @@ -87,27 +97,24 @@ public VectorizedOrcAcidRowBatchReader(InputSplit inputSplit, JobConf conf, } final OrcSplit orcSplit = (OrcSplit) inputSplit; - rbCtx = Utilities.getVectorizedRowBatchCtx(conf); - reporter.setStatus(orcSplit.toString()); - Reader reader = OrcInputFormat.createOrcReaderForSplit(conf, orcSplit); + final Reader reader = OrcInputFormat.createOrcReaderForSplit(conf, orcSplit); Reader.Options readerOptions = OrcInputFormat.createOptionsForReader(conf); readerOptions = OrcRawRecordMerger.createEventOptions(readerOptions); this.offset = orcSplit.getStart(); this.length = orcSplit.getLength(); - // Careful with the range here now, we do not want to read the whole base file like deltas. - this.baseReader = reader.rowsOptions(readerOptions.range(offset, length)); - - // VectorizedRowBatchBase schema is picked up from the baseReader because the SchemaEvolution - // stuff happens at the ORC layer that understands how to map user schema to acid schema. - if (this.baseReader instanceof RecordReaderImpl) { - this.vectorizedRowBatchBase = ((RecordReaderImpl) this.baseReader).createRowBatch(); + if (baseReader == null) { + rbCtx = Utilities.getVectorizedRowBatchCtx(conf); + // Careful with the range here now, we do not want to read the whole base file like deltas. + final RecordReader innerReader = reader.rowsOptions(readerOptions.range(offset, length)); + baseReader = new RecordReaderAdaptor(innerReader, rbCtx); + this.vectorizedRowBatchBase = ((RecordReaderImpl) innerReader).createRowBatch(); } else { - throw new IOException("Failed to create vectorized row batch for the reader of type " - + this.baseReader.getClass().getName()); + this.vectorizedRowBatchBase = baseReader.createValue(); } + this.baseReader = baseReader; int partitionColumnCount = (rbCtx != null) ? rbCtx.getPartitionColumnCount() : 0; if (partitionColumnCount > 0) { @@ -188,7 +195,7 @@ public boolean next(NullWritable key, VectorizedRowBatch value) throws IOExcepti } addPartitionCols = false; } - if (!baseReader.nextBatch(vectorizedRowBatchBase)) { + if (!baseReader.next(null, vectorizedRowBatchBase)) { return false; } } catch (Exception e) { diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java index c21327f517..9ecae50969 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java @@ -2149,7 +2149,10 @@ private static TreeReader createEncodedTreeReader(TypeDescription schema, } else if (batch.hasVectors(columnIndex)) { vectors = batch.getColumnVectors(columnIndex); } else { - throw new AssertionError("Batch has no data for " + columnIndex + ": " + batch); + if (LOG.isDebugEnabled()) { + LOG.debug("Batch has no data for " + columnIndex + ": " + batch); + } + return null; } // EncodedColumnBatch is already decompressed, we don't really need to pass codec. @@ -2712,7 +2715,9 @@ public void seek(PositionProvider[] index) throws IOException { } if (fields != null) { for (TreeReader child : fields) { - child.seek(index); + if (child != null) { + child.seek(index); + } } } } diff --git ql/src/test/queries/clientpositive/llap_acid_fast.q ql/src/test/queries/clientpositive/llap_acid_fast.q new file mode 100644 index 0000000000..b8dbba62c9 --- /dev/null +++ ql/src/test/queries/clientpositive/llap_acid_fast.q @@ -0,0 +1,50 @@ +set hive.mapred.mode=nonstrict; +SET hive.vectorized.execution.enabled=true; + +SET hive.llap.io.enabled=true; + +SET hive.exec.orc.default.buffer.size=32768; +SET hive.exec.orc.default.row.index.stride=1000; +SET hive.optimize.index.filter=true; +set hive.fetch.task.conversion=none; + +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + + +DROP TABLE orc_llap_acid_fast; + +CREATE TABLE orc_llap_acid_fast ( + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE) +partitioned by (csmallint smallint) +clustered by (cint) into 2 buckets stored as orc +TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default'); + +insert into table orc_llap_acid_fast partition (csmallint = 1) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10; +insert into table orc_llap_acid_fast partition (csmallint = 2) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10; +insert into table orc_llap_acid_fast partition (csmallint = 3) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble desc limit 10; + +explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint; +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint; + +insert into table orc_llap_acid_fast partition (csmallint = 1) values (1, 1, 1, 1); + +update orc_llap_acid_fast set cbigint = 2 where cint = 1; + +explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint; +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint; + +DROP TABLE orc_llap_acid_fast; diff --git ql/src/test/results/clientpositive/llap/llap_acid.q.out ql/src/test/results/clientpositive/llap/llap_acid.q.out new file mode 100644 index 0000000000..99db632139 --- /dev/null +++ ql/src/test/results/clientpositive/llap/llap_acid.q.out @@ -0,0 +1,321 @@ +PREHOOK: query: DROP TABLE orc_llap +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orc_llap +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orc_llap ( + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE) +partitioned by (csmallint smallint) +clustered by (cint) into 2 buckets stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_llap +POSTHOOK: query: CREATE TABLE orc_llap ( + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE) +partitioned by (csmallint smallint) +clustered by (cint) into 2 buckets stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_llap +PREHOOK: query: insert into table orc_llap partition (csmallint = 1) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@orc_llap@csmallint=1 +POSTHOOK: query: insert into table orc_llap partition (csmallint = 1) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@orc_llap@csmallint=1 +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=1).cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=1).cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=1).cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=1).cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +PREHOOK: query: insert into table orc_llap partition (csmallint = 2) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@orc_llap@csmallint=2 +POSTHOOK: query: insert into table orc_llap partition (csmallint = 2) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@orc_llap@csmallint=2 +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=2).cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=2).cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=2).cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=2).cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +PREHOOK: query: alter table orc_llap SET TBLPROPERTIES ('transactional'='true') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@orc_llap +PREHOOK: Output: default@orc_llap +POSTHOOK: query: alter table orc_llap SET TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@orc_llap +POSTHOOK: Output: default@orc_llap +PREHOOK: query: insert into table orc_llap partition (csmallint = 3) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble desc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@orc_llap@csmallint=3 +POSTHOOK: query: insert into table orc_llap partition (csmallint = 3) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble desc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@orc_llap@csmallint=3 +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=3).cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=3).cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=3).cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=3).cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +PREHOOK: query: explain +select cint, csmallint, cbigint from orc_llap where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +POSTHOOK: query: explain +select cint, csmallint, cbigint from orc_llap where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orc_llap + filterExpr: cint is not null (type: boolean) + Statistics: Num rows: 20 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: cint (type: int), csmallint (type: smallint), cbigint (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: int) + sort order: ++ + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: int), KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint, csmallint, cbigint from orc_llap where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_llap +PREHOOK: Input: default@orc_llap@csmallint=1 +PREHOOK: Input: default@orc_llap@csmallint=2 +PREHOOK: Input: default@orc_llap@csmallint=3 +#### A masked pattern was here #### +POSTHOOK: query: select cint, csmallint, cbigint from orc_llap where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_llap +POSTHOOK: Input: default@orc_llap@csmallint=1 +POSTHOOK: Input: default@orc_llap@csmallint=2 +POSTHOOK: Input: default@orc_llap@csmallint=3 +#### A masked pattern was here #### +-285355633 1 -1241163445 +-109813638 1 -58941842 +164554497 1 1161977292 +199879534 1 123351087 +246423894 1 -1645852809 +354670578 1 562841852 +455419170 1 1108177470 +665801232 1 480783141 +708885482 1 -1645852809 +-285355633 2 -1241163445 +-109813638 2 -58941842 +164554497 2 1161977292 +199879534 2 123351087 +246423894 2 -1645852809 +354670578 2 562841852 +455419170 2 1108177470 +665801232 2 480783141 +708885482 2 -1645852809 +-923308739 3 -1887561756 +-3728 3 -1887561756 +762 3 -1645852809 +6981 3 -1887561756 +253665376 3 NULL +497728223 3 -1887561756 +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +PREHOOK: query: insert into table orc_llap partition (csmallint = 1) values (1, 1, 1, 1) +PREHOOK: type: QUERY +PREHOOK: Output: default@orc_llap@csmallint=1 +POSTHOOK: query: insert into table orc_llap partition (csmallint = 1) values (1, 1, 1, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@orc_llap@csmallint=1 +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=1).cbigint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=1).cdouble EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=1).cfloat EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: orc_llap PARTITION(csmallint=1).cint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: update orc_llap set cbigint = 2 where cint = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_llap +PREHOOK: Input: default@orc_llap@csmallint=1 +PREHOOK: Input: default@orc_llap@csmallint=2 +PREHOOK: Input: default@orc_llap@csmallint=3 +PREHOOK: Output: default@orc_llap@csmallint=1 +PREHOOK: Output: default@orc_llap@csmallint=2 +PREHOOK: Output: default@orc_llap@csmallint=3 +POSTHOOK: query: update orc_llap set cbigint = 2 where cint = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_llap +POSTHOOK: Input: default@orc_llap@csmallint=1 +POSTHOOK: Input: default@orc_llap@csmallint=2 +POSTHOOK: Input: default@orc_llap@csmallint=3 +POSTHOOK: Output: default@orc_llap@csmallint=1 +POSTHOOK: Output: default@orc_llap@csmallint=2 +POSTHOOK: Output: default@orc_llap@csmallint=3 +PREHOOK: query: explain +select cint, csmallint, cbigint from orc_llap where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +POSTHOOK: query: explain +select cint, csmallint, cbigint from orc_llap where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orc_llap + filterExpr: cint is not null (type: boolean) + Statistics: Num rows: 20 Data size: 376 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: cint (type: int), csmallint (type: smallint), cbigint (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: int) + sort order: ++ + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: int), KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 80 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint, csmallint, cbigint from orc_llap where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_llap +PREHOOK: Input: default@orc_llap@csmallint=1 +PREHOOK: Input: default@orc_llap@csmallint=2 +PREHOOK: Input: default@orc_llap@csmallint=3 +#### A masked pattern was here #### +POSTHOOK: query: select cint, csmallint, cbigint from orc_llap where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_llap +POSTHOOK: Input: default@orc_llap@csmallint=1 +POSTHOOK: Input: default@orc_llap@csmallint=2 +POSTHOOK: Input: default@orc_llap@csmallint=3 +#### A masked pattern was here #### +-285355633 1 -1241163445 +-109813638 1 -58941842 +1 1 2 +164554497 1 1161977292 +199879534 1 123351087 +246423894 1 -1645852809 +354670578 1 562841852 +455419170 1 1108177470 +665801232 1 480783141 +708885482 1 -1645852809 +-285355633 2 -1241163445 +-109813638 2 -58941842 +164554497 2 1161977292 +199879534 2 123351087 +246423894 2 -1645852809 +354670578 2 562841852 +455419170 2 1108177470 +665801232 2 480783141 +708885482 2 -1645852809 +-923308739 3 -1887561756 +-3728 3 -1887561756 +762 3 -1645852809 +6981 3 -1887561756 +253665376 3 NULL +497728223 3 -1887561756 +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +PREHOOK: query: DROP TABLE orc_llap +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orc_llap +PREHOOK: Output: default@orc_llap +POSTHOOK: query: DROP TABLE orc_llap +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orc_llap +POSTHOOK: Output: default@orc_llap diff --git ql/src/test/results/clientpositive/llap/llap_acid_fast.q.out ql/src/test/results/clientpositive/llap/llap_acid_fast.q.out new file mode 100644 index 0000000000..5075354a64 --- /dev/null +++ ql/src/test/results/clientpositive/llap/llap_acid_fast.q.out @@ -0,0 +1,315 @@ +PREHOOK: query: DROP TABLE orc_llap_acid_fast +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orc_llap_acid_fast +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orc_llap_acid_fast ( + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE) +partitioned by (csmallint smallint) +clustered by (cint) into 2 buckets stored as orc +TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_llap_acid_fast +POSTHOOK: query: CREATE TABLE orc_llap_acid_fast ( + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE) +partitioned by (csmallint smallint) +clustered by (cint) into 2 buckets stored as orc +TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_llap_acid_fast +PREHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 1) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 1) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +PREHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 2) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 2) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=2).cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=2).cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=2).cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=2).cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +PREHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 3) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble desc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=3 +POSTHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 3) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble desc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=3 +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=3).cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=3).cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=3).cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=3).cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +PREHOOK: query: explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +POSTHOOK: query: explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orc_llap_acid_fast + filterExpr: cint is not null (type: boolean) + Statistics: Num rows: 247 Data size: 3959 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 247 Data size: 988 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: cint (type: int), csmallint (type: smallint), cbigint (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 247 Data size: 988 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: int) + sort order: ++ + Statistics: Num rows: 247 Data size: 988 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: int), KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 247 Data size: 988 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 247 Data size: 988 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_llap_acid_fast +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +#### A masked pattern was here #### +POSTHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_llap_acid_fast +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +#### A masked pattern was here #### +-285355633 1 -1241163445 +-109813638 1 -58941842 +164554497 1 1161977292 +199879534 1 123351087 +246423894 1 -1645852809 +354670578 1 562841852 +455419170 1 1108177470 +665801232 1 480783141 +708885482 1 -1645852809 +-285355633 2 -1241163445 +-109813638 2 -58941842 +164554497 2 1161977292 +199879534 2 123351087 +246423894 2 -1645852809 +354670578 2 562841852 +455419170 2 1108177470 +665801232 2 480783141 +708885482 2 -1645852809 +-923308739 3 -1887561756 +-3728 3 -1887561756 +762 3 -1645852809 +6981 3 -1887561756 +253665376 3 NULL +497728223 3 -1887561756 +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +PREHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 1) values (1, 1, 1, 1) +PREHOOK: type: QUERY +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 1) values (1, 1, 1, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cbigint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cdouble EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cfloat EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: update orc_llap_acid_fast set cbigint = 2 where cint = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_llap_acid_fast +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=2 +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=3 +POSTHOOK: query: update orc_llap_acid_fast set cbigint = 2 where cint = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_llap_acid_fast +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=3 +PREHOOK: query: explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +POSTHOOK: query: explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orc_llap_acid_fast + filterExpr: cint is not null (type: boolean) + Statistics: Num rows: 424 Data size: 6790 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 424 Data size: 1696 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: cint (type: int), csmallint (type: smallint), cbigint (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 424 Data size: 1696 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: int) + sort order: ++ + Statistics: Num rows: 424 Data size: 1696 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col2 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: int), KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 424 Data size: 1696 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 424 Data size: 1696 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_llap_acid_fast +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +#### A masked pattern was here #### +POSTHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_llap_acid_fast +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +#### A masked pattern was here #### +-285355633 1 -1241163445 +-109813638 1 -58941842 +1 1 2 +164554497 1 1161977292 +199879534 1 123351087 +246423894 1 -1645852809 +354670578 1 562841852 +455419170 1 1108177470 +665801232 1 480783141 +708885482 1 -1645852809 +-285355633 2 -1241163445 +-109813638 2 -58941842 +164554497 2 1161977292 +199879534 2 123351087 +246423894 2 -1645852809 +354670578 2 562841852 +455419170 2 1108177470 +665801232 2 480783141 +708885482 2 -1645852809 +-923308739 3 -1887561756 +-3728 3 -1887561756 +762 3 -1645852809 +6981 3 -1887561756 +253665376 3 NULL +497728223 3 -1887561756 +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +PREHOOK: query: DROP TABLE orc_llap_acid_fast +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orc_llap_acid_fast +PREHOOK: Output: default@orc_llap_acid_fast +POSTHOOK: query: DROP TABLE orc_llap_acid_fast +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orc_llap_acid_fast +POSTHOOK: Output: default@orc_llap_acid_fast diff --git ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index 8af84dce19..ac393edcee 100644 --- ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -565,9 +565,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -576,14 +576,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 50 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 1100 - SELECTED_ROWGROUPS: 2 50 PREHOOK: query: select count(*) from orc_ppd where t between 60 and 80 PREHOOK: type: QUERY @@ -663,9 +655,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17479 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -674,9 +666,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 2 - SELECTED_ROWGROUPS: 0 0 PREHOOK: query: select count(*) from orc_ppd where s > "zzz" PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index 77b7f5a2f7..ac7103a68f 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -203,9 +203,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 16673 + HDFS_BYTES_READ: 16649 HDFS_BYTES_WRITTEN: 104 - HDFS_READ_OPS: 5 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -214,12 +214,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_MISS: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 2100 PREHOOK: query: select count(*) from orc_ppd where t > 127 PREHOOK: type: QUERY @@ -240,7 +234,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1344 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -251,25 +245,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 524288 - ALLOCATED_USED_BYTES: 269 - CACHE_MISS_BYTES: 249 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 8 PREHOOK: query: select count(*) from orc_ppd where t <=> 50 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -278,23 +262,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 22 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 22 PREHOOK: query: select count(*) from orc_ppd where t <=> 100 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -303,23 +279,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 16 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 16 PREHOOK: query: select count(*) from orc_ppd where t = "54" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -328,23 +296,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 18 PREHOOK: query: select count(*) from orc_ppd where t = -10.0 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -353,23 +313,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 1 PREHOOK: query: select count(*) from orc_ppd where t = cast(53 as float) PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -378,23 +330,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 32 PREHOOK: query: select count(*) from orc_ppd where t = cast(53 as double) PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -403,23 +347,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 32 PREHOOK: query: select count(*) from orc_ppd where t < 100 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 104 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -428,23 +364,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1697 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 1697 PREHOOK: query: select count(*) from orc_ppd where t < 100 and t > 98 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -453,23 +381,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 12 PREHOOK: query: select count(*) from orc_ppd where t <= 100 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 104 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -478,23 +398,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1713 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 1713 PREHOOK: query: select count(*) from orc_ppd where t is null PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -503,23 +415,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 6 PREHOOK: query: select count(*) from orc_ppd where t in (5, 120) PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -528,23 +432,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 50 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 1100 - SELECTED_ROWGROUPS: 2 50 PREHOOK: query: select count(*) from orc_ppd where t between 60 and 80 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 103 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -553,14 +449,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 318 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 318 PREHOOK: query: select count(*) from orc_ppd where t = -100 PREHOOK: type: QUERY @@ -595,9 +483,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17479 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -606,18 +494,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 2 - SELECTED_ROWGROUPS: 0 0 PREHOOK: query: select count(*) from orc_ppd where t IN (-100, 125, 200) PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17479 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -626,9 +511,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 2 - SELECTED_ROWGROUPS: 0 0 PREHOOK: query: select count(*) from orc_ppd where s > "zzz" PREHOOK: type: QUERY @@ -649,7 +531,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 6786 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -660,26 +542,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 786432 - ALLOCATED_USED_BYTES: 11299 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 3980 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 2 PREHOOK: query: select count(*) from orc_ppd where s <=> "zach zipper" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -688,23 +559,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 6 PREHOOK: query: select count(*) from orc_ppd where s <=> "" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -713,14 +576,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 6 PREHOOK: query: select count(*) from orc_ppd where s is null PREHOOK: type: QUERY @@ -741,9 +596,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 104 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -752,23 +607,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 2100 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as char(50)) PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 18384 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -777,18 +624,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 2 - SELECTED_ROWGROUPS: 0 0 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as char(10)) PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -797,23 +641,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as varchar(10)) PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -822,23 +658,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as varchar(50)) PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -847,23 +675,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 2 PREHOOK: query: select count(*) from orc_ppd where s < "b" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -872,23 +692,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 81 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 81 PREHOOK: query: select count(*) from orc_ppd where s > "alice" and s < "bob" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -897,23 +709,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 74 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 74 PREHOOK: query: select count(*) from orc_ppd where s in ("alice allen", "") PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -922,23 +726,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 12 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 12 PREHOOK: query: select count(*) from orc_ppd where s between "" and "alice allen" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -947,23 +743,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 13 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 13 PREHOOK: query: select count(*) from orc_ppd where s between "zz" and "zzz" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -972,23 +760,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 100 - SELECTED_ROWGROUPS: 1 1 PREHOOK: query: select count(*) from orc_ppd where s between "zach zipper" and "zzz" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -997,23 +777,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 7 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 1100 - SELECTED_ROWGROUPS: 2 7 PREHOOK: query: select count(*) from orc_ppd where s = "hello world" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 18384 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1022,18 +794,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 2 - SELECTED_ROWGROUPS: 0 0 PREHOOK: query: select count(*) from orc_ppd where s <=> "apache hive" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 18384 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1042,18 +811,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 2 - SELECTED_ROWGROUPS: 0 0 PREHOOK: query: select count(*) from orc_ppd where s IN ("a", "z") PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 18384 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1062,18 +828,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 2 - SELECTED_ROWGROUPS: 0 0 PREHOOK: query: select count(*) from orc_ppd where s = "sarah ovid" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1082,23 +845,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 100 - SELECTED_ROWGROUPS: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = "wendy king" PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1107,23 +862,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 1100 - SELECTED_ROWGROUPS: 2 6 PREHOOK: query: select count(*) from orc_ppd where s = "wendy king" and t < 0 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 23419 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1132,23 +879,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4229 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = "wendy king" and t > 100 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 23419 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1157,21 +896,13 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4229 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 100 - SELECTED_ROWGROUPS: 1 2 PREHOOK: query: select count(*) from orc_ppd where f=74.72 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 7718 + HDFS_BYTES_READ: 21458 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -1182,51 +913,32 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 262144 - ALLOCATED_USED_BYTES: 8400 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 4809 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 2 PREHOOK: query: select count(*) from orc_ppd where f=74.72 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 23336 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4809 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 2 PREHOOK: query: select count(*) from orc_ppd where f=74.72 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 21458 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1235,39 +947,23 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4809 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 2 PREHOOK: query: select count(*) from orc_ppd where f=74.72 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 23336 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4809 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 2 PREHOOK: query: create temporary table tmp_orcppd stored as orc @@ -1278,8 +974,6 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@tmp_orcppd -Stage-1 LLAP IO COUNTERS: - SELECTED_ROWGROUPS: 3 PREHOOK: query: insert into table tmp_orcppd values(null, null, null, null, null, null, null, null, null, null) @@ -1309,8 +1003,6 @@ PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default PREHOOK: Output: default@tmp_orcppd -Stage-1 LLAP IO COUNTERS: - SELECTED_ROWGROUPS: 3 PREHOOK: query: insert into table tmp_orcppd values(null, null) PREHOOK: type: QUERY @@ -1351,5 +1043,3 @@ PREHOOK: Input: default@alltypesorc PREHOOK: Input: default@tmp_orcppd PREHOOK: Output: database:default PREHOOK: Output: default@tbl_orcppd_3_1 -Stage-1 LLAP IO COUNTERS: - SELECTED_ROWGROUPS: 4 diff --git ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out index b799527e30..3e4804f764 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out @@ -203,9 +203,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17008 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 6 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -214,15 +214,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 524288 - ALLOCATED_USED_BYTES: 269 - CACHE_MISS_BYTES: 249 - METADATA_CACHE_MISS: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 0 PREHOOK: query: select count(*) from orc_ppd where t > 127 PREHOOK: type: QUERY @@ -243,9 +234,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -254,23 +245,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 8 PREHOOK: query: select count(*) from orc_ppd where t = 55 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -279,23 +262,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 8 PREHOOK: query: select count(*) from orc_ppd where t = 54 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -304,23 +279,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 18 PREHOOK: query: select count(*) from orc_ppd where t = 54 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -329,14 +296,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 18 PREHOOK: query: alter table orc_ppd change column t t smallint PREHOOK: type: ALTERTABLE_RENAMECOL @@ -347,9 +306,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -358,14 +317,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 0 PREHOOK: query: select count(*) from orc_ppd where t > 127 PREHOOK: type: QUERY @@ -386,9 +337,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -397,23 +348,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 8 PREHOOK: query: select count(*) from orc_ppd where t = 55 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -422,23 +365,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 8 PREHOOK: query: select count(*) from orc_ppd where t = 54 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -447,23 +382,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 18 PREHOOK: query: select count(*) from orc_ppd where t = 54 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -472,14 +399,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 18 PREHOOK: query: alter table orc_ppd change column t t int PREHOOK: type: ALTERTABLE_RENAMECOL @@ -490,9 +409,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -501,14 +420,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 0 PREHOOK: query: select count(*) from orc_ppd where t > 127 PREHOOK: type: QUERY @@ -529,9 +440,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -540,23 +451,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 8 PREHOOK: query: select count(*) from orc_ppd where t = 55 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -565,23 +468,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 8 PREHOOK: query: select count(*) from orc_ppd where t = 54 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -590,23 +485,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 18 PREHOOK: query: select count(*) from orc_ppd where t = 54 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -615,14 +502,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 18 PREHOOK: query: alter table orc_ppd change column t t bigint PREHOOK: type: ALTERTABLE_RENAMECOL @@ -633,9 +512,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -644,14 +523,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 0 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 0 PREHOOK: query: select count(*) from orc_ppd where t > 127 PREHOOK: type: QUERY @@ -672,9 +543,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -683,23 +554,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 8 PREHOOK: query: select count(*) from orc_ppd where t = 55 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -708,23 +571,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 1 - NUM_VECTOR_BATCHES: 1 - ROWS_EMITTED: 1000 - SELECTED_ROWGROUPS: 1 8 PREHOOK: query: select count(*) from orc_ppd where t = 54 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 16898 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -733,23 +588,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 18 PREHOOK: query: select count(*) from orc_ppd where t = 54 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 17728 HDFS_BYTES_WRITTEN: 102 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -758,14 +605,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 249 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 2 - NUM_VECTOR_BATCHES: 2 - ROWS_EMITTED: 2000 - SELECTED_ROWGROUPS: 2 18 PREHOOK: query: alter table orc_ppd change column t t string PREHOOK: type: ALTERTABLE_RENAMECOL @@ -787,8 +626,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1566 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 1566 PREHOOK: query: select count(*) from orc_ppd where t > '127' PREHOOK: type: QUERY @@ -806,8 +643,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1566 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 1566 PREHOOK: query: select count(*) from orc_ppd where t = '55' PREHOOK: type: QUERY @@ -825,8 +660,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 8 PREHOOK: query: select count(*) from orc_ppd where t = '55' PREHOOK: type: QUERY @@ -844,8 +677,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 8 PREHOOK: query: select count(*) from orc_ppd where t = '54' PREHOOK: type: QUERY @@ -863,8 +694,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 18 PREHOOK: query: select count(*) from orc_ppd where t = '54' PREHOOK: type: QUERY @@ -882,15 +711,13 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 18 PREHOOK: query: select count(*) from orc_ppd where f = 74.72 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 5287 + HDFS_BYTES_READ: 21458 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -901,42 +728,23 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 262144 - ALLOCATED_USED_BYTES: 8400 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 4809 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 2 PREHOOK: query: select count(*) from orc_ppd where f = 74.72 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 23336 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 4809 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 2 PREHOOK: query: alter table orc_ppd change column f f double PREHOOK: type: ALTERTABLE_RENAMECOL @@ -958,8 +766,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 2 PREHOOK: query: select count(*) from orc_ppd where f = 74.72 PREHOOK: type: QUERY @@ -977,8 +783,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 2 PREHOOK: query: alter table orc_ppd change column f f string PREHOOK: type: ALTERTABLE_RENAMECOL @@ -1000,8 +804,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 2 PREHOOK: query: select count(*) from orc_ppd where f = '74.72' PREHOOK: type: QUERY @@ -1019,15 +821,13 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = 'bob davidson' PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 4577 + HDFS_BYTES_READ: 20629 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -1038,26 +838,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 786432 - ALLOCATED_USED_BYTES: 11299 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 3980 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 6 PREHOOK: query: select count(*) from orc_ppd where s = 'bob davidson' PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1066,14 +855,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 6 PREHOOK: query: alter table orc_ppd change column s s char(50) PREHOOK: type: ALTERTABLE_RENAMECOL @@ -1095,8 +876,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 6 PREHOOK: query: select count(*) from orc_ppd where s = 'bob davidson' PREHOOK: type: QUERY @@ -1114,8 +893,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 6 PREHOOK: query: alter table orc_ppd change column s s varchar(50) PREHOOK: type: ALTERTABLE_RENAMECOL @@ -1126,9 +903,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 20629 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1137,23 +914,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 6 PREHOOK: query: select count(*) from orc_ppd where s = 'bob davidson' PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1162,14 +931,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 6 PREHOOK: query: alter table orc_ppd change column s s char(50) PREHOOK: type: ALTERTABLE_RENAMECOL @@ -1191,8 +952,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 6 PREHOOK: query: select count(*) from orc_ppd where s = 'bob davidson' PREHOOK: type: QUERY @@ -1210,8 +969,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 6 PREHOOK: query: alter table orc_ppd change column s s string PREHOOK: type: ALTERTABLE_RENAMECOL @@ -1222,9 +979,9 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 20629 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1233,23 +990,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 6 PREHOOK: query: select count(*) from orc_ppd where s = 'bob davidson' PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 22364 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: @@ -1258,14 +1007,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 6 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 3980 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 6 PREHOOK: query: alter table orc_ppd add columns (boo boolean) PREHOOK: type: ALTERTABLE_ADDCOLS @@ -1276,7 +1017,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 2780 + HDFS_BYTES_READ: 18747 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -1287,17 +1028,6 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 -Stage-1 LLAP IO COUNTERS: - ALLOCATED_BYTES: 524288 - ALLOCATED_USED_BYTES: 4201 - CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 2098 - METADATA_CACHE_HIT: 1 - METADATA_CACHE_MISS: 1 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 4 PREHOOK: query: select count(*) from orc_ppd where si = 442 or boo is not null or boo = false PREHOOK: type: QUERY @@ -1315,33 +1045,23 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 4 PREHOOK: query: select count(*) from orc_ppd where si = 442 PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 0 + HDFS_BYTES_READ: 20073 HDFS_BYTES_WRITTEN: 101 - HDFS_READ_OPS: 2 + HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 2 Stage-1 HIVE COUNTERS: CREATED_FILES: 1 DESERIALIZE_ERRORS: 0 - RECORDS_IN_Map_1: 2100 + RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 -Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 2098 - CACHE_MISS_BYTES: 0 - METADATA_CACHE_HIT: 2 - NUM_DECODED_BATCHES: 3 - NUM_VECTOR_BATCHES: 3 - ROWS_EMITTED: 2100 - SELECTED_ROWGROUPS: 3 4 PREHOOK: query: select count(*) from orc_ppd where si = 442 or boo is not null or boo = false PREHOOK: type: QUERY @@ -1359,6 +1079,4 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 4 -Stage-1 LLAP IO COUNTERS: - METADATA_CACHE_HIT: 1 4 diff --git ql/src/test/results/clientpositive/llap_acid_fast.q.out ql/src/test/results/clientpositive/llap_acid_fast.q.out new file mode 100644 index 0000000000..d93aa4b68a --- /dev/null +++ ql/src/test/results/clientpositive/llap_acid_fast.q.out @@ -0,0 +1,299 @@ +PREHOOK: query: DROP TABLE orc_llap_acid_fast +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE orc_llap_acid_fast +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE orc_llap_acid_fast ( + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE) +partitioned by (csmallint smallint) +clustered by (cint) into 2 buckets stored as orc +TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_llap_acid_fast +POSTHOOK: query: CREATE TABLE orc_llap_acid_fast ( + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE) +partitioned by (csmallint smallint) +clustered by (cint) into 2 buckets stored as orc +TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_llap_acid_fast +PREHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 1) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 1) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +PREHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 2) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 2) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble asc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=2).cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=2).cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=2).cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=2).cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +PREHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 3) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble desc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=3 +POSTHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 3) +select cint, cbigint, cfloat, cdouble from alltypesorc order by cdouble desc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=3 +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=3).cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=3).cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=3).cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=3).cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +PREHOOK: query: explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +POSTHOOK: query: explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: orc_llap_acid_fast + filterExpr: cint is not null (type: boolean) + Statistics: Num rows: 465 Data size: 5585 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 465 Data size: 5585 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int), csmallint (type: smallint), cbigint (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 465 Data size: 5585 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: int) + sort order: ++ + Statistics: Num rows: 465 Data size: 5585 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: vectorized + LLAP IO: may be used (ACID table) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: int), KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 465 Data size: 5585 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 465 Data size: 5585 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_llap_acid_fast +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +#### A masked pattern was here #### +POSTHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_llap_acid_fast +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +#### A masked pattern was here #### +-285355633 1 -1241163445 +-109813638 1 -58941842 +164554497 1 1161977292 +199879534 1 123351087 +246423894 1 -1645852809 +354670578 1 562841852 +455419170 1 1108177470 +665801232 1 480783141 +708885482 1 -1645852809 +-285355633 2 -1241163445 +-109813638 2 -58941842 +164554497 2 1161977292 +199879534 2 123351087 +246423894 2 -1645852809 +354670578 2 562841852 +455419170 2 1108177470 +665801232 2 480783141 +708885482 2 -1645852809 +-923308739 3 -1887561756 +-3728 3 -1887561756 +762 3 -1645852809 +6981 3 -1887561756 +253665376 3 NULL +497728223 3 -1887561756 +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +PREHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 1) values (1, 1, 1, 1) +PREHOOK: type: QUERY +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: query: insert into table orc_llap_acid_fast partition (csmallint = 1) values (1, 1, 1, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cbigint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cdouble EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cfloat EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: orc_llap_acid_fast PARTITION(csmallint=1).cint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: update orc_llap_acid_fast set cbigint = 2 where cint = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_llap_acid_fast +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=2 +PREHOOK: Output: default@orc_llap_acid_fast@csmallint=3 +POSTHOOK: query: update orc_llap_acid_fast set cbigint = 2 where cint = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_llap_acid_fast +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=3 +PREHOOK: query: explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +POSTHOOK: query: explain +select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: orc_llap_acid_fast + filterExpr: cint is not null (type: boolean) + Statistics: Num rows: 642 Data size: 7718 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: cint is not null (type: boolean) + Statistics: Num rows: 642 Data size: 7718 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int), csmallint (type: smallint), cbigint (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 642 Data size: 7718 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: smallint), _col0 (type: int) + sort order: ++ + Statistics: Num rows: 642 Data size: 7718 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: bigint) + Execution mode: vectorized + LLAP IO: may be used (ACID table) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: int), KEY.reducesinkkey0 (type: smallint), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 642 Data size: 7718 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 642 Data size: 7718 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_llap_acid_fast +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +PREHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +#### A masked pattern was here #### +POSTHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order +by csmallint, cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_llap_acid_fast +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=1 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=2 +POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=3 +#### A masked pattern was here #### +-285355633 1 -1241163445 +-109813638 1 -58941842 +1 1 2 +164554497 1 1161977292 +199879534 1 123351087 +246423894 1 -1645852809 +354670578 1 562841852 +455419170 1 1108177470 +665801232 1 480783141 +708885482 1 -1645852809 +-285355633 2 -1241163445 +-109813638 2 -58941842 +164554497 2 1161977292 +199879534 2 123351087 +246423894 2 -1645852809 +354670578 2 562841852 +455419170 2 1108177470 +665801232 2 480783141 +708885482 2 -1645852809 +-923308739 3 -1887561756 +-3728 3 -1887561756 +762 3 -1645852809 +6981 3 -1887561756 +253665376 3 NULL +497728223 3 -1887561756 +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +528534767 3 NULL +PREHOOK: query: DROP TABLE orc_llap_acid_fast +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orc_llap_acid_fast +PREHOOK: Output: default@orc_llap_acid_fast +POSTHOOK: query: DROP TABLE orc_llap_acid_fast +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orc_llap_acid_fast +POSTHOOK: Output: default@orc_llap_acid_fast