diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 4aa4474..2ffc488 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -156,6 +156,7 @@ //columnName to column position map // private final Map columnMap; private int firstOutputColumnIndex; + private boolean reuseScratchColumns = false; public enum HiveVectorAdaptorUsageMode { NONE, @@ -177,6 +178,7 @@ private void setHiveConfVars(HiveConf hiveConf) { private void copyHiveConfVars(VectorizationContext vContextEnvironment) { hiveVectorAdaptorUsageMode = vContextEnvironment.hiveVectorAdaptorUsageMode; + reuseScratchColumns = vContextEnvironment.reuseScratchColumns; } // Convenient constructor for initial batch creation takes @@ -195,7 +197,7 @@ public VectorizationContext(String contextName, List initialColumnNames, projectionColumnMap.put(projectionColumnNames.get(i), i); } int firstOutputColumnIndex = projectedColumns.size(); - this.ocm = new OutputColumnManager(firstOutputColumnIndex); + this.ocm = new OutputColumnManager(firstOutputColumnIndex, reuseScratchColumns); this.firstOutputColumnIndex = firstOutputColumnIndex; vMap = new VectorExpressionDescriptor(); @@ -224,7 +226,7 @@ public VectorizationContext(String contextName, HiveConf hiveConf) { projectedColumns = new ArrayList(); projectionColumnNames = new ArrayList(); projectionColumnMap = new HashMap(); - this.ocm = new OutputColumnManager(0); + this.ocm = new OutputColumnManager(0, reuseScratchColumns); this.firstOutputColumnIndex = 0; vMap = new VectorExpressionDescriptor(); @@ -271,7 +273,7 @@ public void addInitialColumn(String columnName) { // columns have been added. public void finishedAddingInitialColumns() { int firstOutputColumnIndex = projectedColumns.size(); - this.ocm = new OutputColumnManager(firstOutputColumnIndex); + this.ocm = new OutputColumnManager(firstOutputColumnIndex, reuseScratchColumns); this.firstOutputColumnIndex = firstOutputColumnIndex; } @@ -424,11 +426,17 @@ protected int getInputColumnIndex(ExprNodeColumnDesc colExpr) throws HiveExcepti private static class OutputColumnManager { private final int initialOutputCol; private int outputColCount = 0; + private boolean reuseScratchColumns = true; protected OutputColumnManager(int initialOutputCol) { this.initialOutputCol = initialOutputCol; } + protected OutputColumnManager(int initialOutputCol, boolean reuseScratchColumns) { + this.initialOutputCol = initialOutputCol; + this.reuseScratchColumns = reuseScratchColumns; + } + //The complete list of output columns. These should be added to the //Vectorized row batch for processing. The index in the row batch is //equal to the index in this array plus initialOutputCol. @@ -479,13 +487,16 @@ private int allocateOutputColumnInternal(String columnType) { } void freeOutputColumn(int index) { - if (initialOutputCol < 0) { + if (initialOutputCol < 0 || reuseScratchColumns == false) { // This is a test return; } - int colIndex = index-initialOutputCol; + int colIndex = index - initialOutputCol; if (colIndex >= 0) { - usedOutputColumns.remove(index-initialOutputCol); + if (!usedOutputColumns.remove(index - initialOutputCol)) { + LOG.warn("Removed a non-existent output column " + index, new Exception( + "Missing output col")); + } } }