diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 3295d1dbc5..efe56c3d19 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3770,6 +3770,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal false, "Use Tez cartesian product edge to speed up cross product"), // The default is different on the client and server, so it's null here. LLAP_IO_ENABLED("hive.llap.io.enabled", null, "Whether the LLAP IO layer is enabled."), + LLAP_IO_ROW_WRAPPER_ENABLED("hive.llap.io.row.wrapper.enabled", true, "Whether the LLAP IO row wrapper is enabled for non-vectorized queries."), LLAP_IO_ACID_ENABLED("hive.llap.io.acid", true, "Whether the LLAP IO layer is enabled for ACID."), LLAP_IO_TRACE_SIZE("hive.llap.io.trace.size", "2Mb", new SizeValidator(0L, true, (long)Integer.MAX_VALUE, false), diff --git llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java index 6d29163fbf..40f7c8385d 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hive.llap.io.api.impl; import org.apache.hadoop.hive.ql.io.BatchToRowInputFormat; - +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import java.io.IOException; @@ -114,7 +114,10 @@ // For non-vectorized operator case, wrap the reader if possible. RecordReader result = rr; if (!Utilities.getIsVectorized(job)) { - result = wrapLlapReader(tableIncludedCols, rr, split); + result = null; + if (HiveConf.getBoolVar(job, ConfVars.LLAP_IO_ROW_WRAPPER_ENABLED)) { + result = wrapLlapReader(tableIncludedCols, rr, split); + } if (result == null) { // Cannot wrap a reader for non-vectorized pipeline. return sourceInputFormat.getRecordReader(split, job, reporter); diff --git llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java index 6897336702..cb57a114b3 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java @@ -178,8 +178,8 @@ private LlapRecordReader(MapWork mapWork, JobConf job, FileSplit split, this.isVectorized = HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED); if (isAcidScan) { - this.acidReader = new VectorizedOrcAcidRowBatchReader((OrcSplit) split, jobConf, Reporter.NULL, null, rbCtx, - true); + this.acidReader = new VectorizedOrcAcidRowBatchReader( + (OrcSplit) split, jobConf, Reporter.NULL, null, rbCtx, true); } // Create the consumer of encoded data; it will coordinate decoding to CVBs. diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java index d2e1a68312..a4568de696 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java @@ -324,8 +324,9 @@ static boolean canUseLlapForAcid(OrcSplit split, boolean hasDeletes, Configurati private static boolean needSyntheticRowIds(boolean isOriginal, boolean hasDeletes, boolean rowIdProjected) { return isOriginal && (hasDeletes || rowIdProjected); } + private static boolean areRowIdsProjected(VectorizedRowBatchCtx rbCtx) { - if(rbCtx.getVirtualColumnCount() == 0) { + if (rbCtx.getVirtualColumnCount() == 0) { return false; } for(VirtualColumn vc : rbCtx.getNeededVirtualColumns()) { @@ -474,6 +475,7 @@ public boolean next(NullWritable key, VectorizedRowBatch value) throws IOExcepti } } if (rowIdProjected) { + // TODO: could we just do: int ix = rbCtx.findVirtualColumnNum(VirtualColumn.ROWID); value.cols[ix] = recordIdColumnVector; rbCtx.setRecordIdColumnVector(recordIdColumnVector); } progress = baseReader.getProgress(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java index 512f1ff3da..15907362f2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -34,6 +34,7 @@ import org.antlr.runtime.tree.Tree; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -519,6 +520,8 @@ private ReparseResult parseRewrittenQuery(StringBuilder rewrittenQueryStr, Strin // references. // todo: this may be a perf issue as it prevents the optimizer.. or not HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); + // Disable LLAP IO wrapper; doesn't propagate extra ACID columns correctly. + HiveConf.setBoolVar(conf, ConfVars.LLAP_IO_ROW_WRAPPER_ENABLED, false); // Parse the rewritten query string Context rewrittenCtx; try {