diff --git ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java index 731b686..9f8554c 100644 --- ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java +++ ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java @@ -261,13 +261,11 @@ {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "GreaterEqual", ">="}, {"FilterStringColumnBetween", ""}, - {"FilterStringColumnBetween", "!"}, - {"FilterTruncStringColumnBetween", "VarChar", ""}, - {"FilterTruncStringColumnBetween", "VarChar", "!"}, - {"FilterTruncStringColumnBetween", "Char", ""}, - {"FilterTruncStringColumnBetween", "Char", "!"}, + + {"TruncStringColumnBetween", "VarChar", ""}, + {"TruncStringColumnBetween", "Char", ""}, {"StringGroupColumnCompareStringGroupScalarBase", "Equal", "=="}, {"StringGroupColumnCompareStringGroupScalarBase", "NotEqual", "!="}, @@ -416,11 +414,10 @@ {"FilterColumnBetween", "long", ""}, {"FilterColumnBetween", "double", ""}, - {"FilterColumnBetween", "long", "!"}, - {"FilterColumnBetween", "double", "!"}, + {"FilterDecimalColumnBetween", ""}, - {"FilterDecimalColumnBetween", ""}, - {"FilterDecimalColumnBetween", "!"}, + {"ColumnBetween", "long", ""}, + {"ColumnBetween", "double", ""}, {"ColumnCompareColumn", "Equal", "long", "double", "=="}, {"ColumnCompareColumn", "Equal", "double", "double", "=="}, @@ -724,6 +721,8 @@ private void generate() throws Exception { generateFilterScalarCompareColumn(tdesc); } else if (tdesc[0].equals("FilterColumnBetween")) { generateFilterColumnBetween(tdesc); + } else if (tdesc[0].equals("ColumnBetween")) { + generateColumnBetween(tdesc); } else if (tdesc[0].equals("ScalarArithmeticColumn") || tdesc[0].equals("ScalarDivideColumn")) { generateScalarArithmeticColumn(tdesc); } else if (tdesc[0].equals("FilterColumnCompareColumn")) { @@ -762,6 +761,8 @@ private void generate() throws Exception { generateFilterStringColumnBetween(tdesc); } else if (tdesc[0].equals("FilterTruncStringColumnBetween")) { generateFilterTruncStringColumnBetween(tdesc); + } else if (tdesc[0].equals("TruncStringColumnBetween")) { + generateTruncStringColumnBetween(tdesc); } else if (tdesc[0].equals("FilterDecimalColumnBetween")) { generateFilterDecimalColumnBetween(tdesc); } else if (tdesc[0].equals("StringGroupColumnCompareStringGroupScalarBase")) { @@ -851,6 +852,33 @@ private void generateFilterTruncStringColumnBetween(String[] tdesc) throws IOExc className, templateString); } + private void generateTruncStringColumnBetween(String[] tdesc) throws IOException { + String truncStringTypeName = tdesc[1]; + String truncStringHiveType; + String truncStringHiveGetBytes; + if (truncStringTypeName == "Char") { + truncStringHiveType = "HiveChar"; + truncStringHiveGetBytes = "getStrippedValue().getBytes()"; + } else if (truncStringTypeName == "VarChar") { + truncStringHiveType = "HiveVarchar"; + truncStringHiveGetBytes = "getValue().getBytes()"; + } else { + throw new Error("Unsupported string type: " + truncStringTypeName); + } + String optionalNot = tdesc[2]; + String className = truncStringTypeName + "ColumnBetween"; + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", truncStringTypeName); + templateString = templateString.replaceAll("", truncStringHiveType); + templateString = templateString.replaceAll("", truncStringHiveGetBytes); + templateString = templateString.replaceAll("", className); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateFilterDecimalColumnBetween(String[] tdesc) throws IOException { String optionalNot = tdesc[1]; String className = "FilterDecimalColumn" + (optionalNot.equals("!") ? "Not" : "") @@ -885,6 +913,24 @@ private void generateFilterColumnBetween(String[] tdesc) throws Exception { className, templateString); } + private void generateColumnBetween(String[] tdesc) throws Exception { + String operandType = tdesc[1]; + String optionalNot = tdesc[2]; + + String className = getCamelCaseType(operandType) + "ColumnBetween"; + String inputColumnVectorType = getColumnVectorType(operandType); + + // Read the template into a string, expand it, and write it. + File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt")); + String templateString = readFile(templateFile); + templateString = templateString.replaceAll("", className); + templateString = templateString.replaceAll("", inputColumnVectorType); + templateString = templateString.replaceAll("", operandType); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); + } + private void generateColumnCompareColumn(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; diff --git ql/src/gen/vectorization/ExpressionTemplates/ColumnBetween.txt ql/src/gen/vectorization/ExpressionTemplates/ColumnBetween.txt new file mode 100644 index 0000000..0aa72a4 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/ColumnBetween.txt @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +/** + * Generated from template ColumnBetween.txt, which covers BETWEEN filter + * expressions where a column is between one scalar and another. + * Output is not produced in a separate column. The selected vector of the input + * {@link VectorizedRowBatch} is updated for in-place filtering. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + int colNum; + int outputColumn; + + // The comparison is of the form "column BETWEEN leftValue AND rightValue" + private leftValue; + private rightValue; + + public (int colNum, leftValue, rightValue, int outputColumn) { + this.colNum = colNum; + this.leftValue = leftValue; + this.rightValue = rightValue; + this.outputColumn = outputColumn; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + inputColVector = () batch.cols[colNum]; + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; + + final int[] sel = batch.selected; + final boolean[] nullPos = inputColVector.isNull; + final boolean[] outNulls = outputColVector.isNull; + final int n = batch.size; + final [] vector = inputColVector.vector; + final long[] outputVector = outputColVector.vector; + + outputColVector.isRepeating = false; + outputColVector.noNulls = inputColVector.noNulls; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.isRepeating) { + outNulls[0] = nullPos[0]; + outputColVector.isRepeating = true; + } else if (!inputColVector.noNulls) { + System.arraycopy(nullPos, 0, outNulls, 0, nullPos.length); + } + + if (inputColVector.noNulls) { + if (inputColVector.isRepeating) { + outputVector[0] = leftValue <= vector[0] && vector[0] <= rightValue ? 1 : 0; + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = leftValue <= vector[i] && vector[i] <= rightValue ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = leftValue <= vector[i] && vector[i] <= rightValue ? 1 : 0; + } + } + } else { + if (inputColVector.isRepeating) { + if (!nullPos[0]) { + outputVector[0] = leftValue <= vector[0] && vector[0] <= rightValue ? 1 : 0; + } + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + outputVector[i] = leftValue <= vector[i] && vector[i] <= rightValue ? 1 : 0; + } + } + } else { + for(int i = 0; i != n; i++) { + if (!nullPos[i]) { + outputVector[i] = leftValue <= vector[i] && vector[i] <= rightValue ? 1 : 0; + } + } + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public getLeftValue() { + return leftValue; + } + + public void setLeftValue( value) { + this.leftValue = value; + } + + public getRightValue() { + return rightValue; + } + + public void setRightValue( value) { + this.rightValue = value; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + // This VectorExpression (BETWEEN) is a special case, so don't return a descriptor. + return null; + } +} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetween.txt ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetween.txt index e9aaaf2..7c5615d 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetween.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterColumnBetween.txt @@ -171,7 +171,7 @@ public class extends VectorExpression { } public void setRightValue( value) { - this.leftValue = value; + this.rightValue = value; } @Override diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetween.txt ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetween.txt index 829b9ca..48d6028 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetween.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnBetween.txt @@ -173,7 +173,7 @@ public class extends VectorExpression { } public void setRightValue(Decimal128 value) { - this.leftValue = value; + this.rightValue = value; } @Override diff --git ql/src/gen/vectorization/ExpressionTemplates/TruncStringColumnBetween.txt ql/src/gen/vectorization/ExpressionTemplates/TruncStringColumnBetween.txt new file mode 100644 index 0000000..6a76b02 --- /dev/null +++ ql/src/gen/vectorization/ExpressionTemplates/TruncStringColumnBetween.txt @@ -0,0 +1,182 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; + +import org.apache.hadoop.hive.common.type.; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +/** + * Generated from template ColumnBetween.txt, which covers BETWEEN filter + * expressions where a column is between one scalar and another. + * Output is not produced in a separate column. The selected vector of the input + * {@link VectorizedRowBatch} is updated for in-place filtering. + */ +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + int colNum; + int outputColumn; + + // The comparison is of the form "column BETWEEN leftValue AND rightValue" + private byte[] left; + private byte[] right; + + public (int colNum, left, right, int outputColumn) { + this.colNum = colNum; + this.left = left.; + this.right = right.; + this.outputColumn = outputColumn; + } + + public () { + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + final BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; + final LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; + + final int[] length = inputColVector.length; + final int[] start = inputColVector.start; + + final int[] sel = batch.selected; + final boolean[] nullPos = inputColVector.isNull; + final boolean[] outNulls = outputColVector.isNull; + final int n = batch.size; + final byte[][] vector = inputColVector.vector; + final long[] outputVector = outputColVector.vector; + + outputColVector.isRepeating = false; + outputColVector.noNulls = inputColVector.noNulls; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.isRepeating) { + outNulls[0] = nullPos[0]; + outputColVector.isRepeating = true; + } else if (!inputColVector.noNulls) { + System.arraycopy(nullPos, 0, outNulls, 0, nullPos.length); + } + + if (inputColVector.noNulls) { + if (inputColVector.isRepeating) { + outputVector[0] = + (StringExpr.compare(left, 0, left.length, vector[0], start[0], length[0]) <= 0 && + StringExpr.compare(vector[0], start[0], length[0], right, 0, right.length) <= 0) ? 1 : 0; + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = + (StringExpr.compare(left, 0, left.length, vector[i], start[i], length[i]) <= 0 && + StringExpr.compare(vector[i], start[i], length[i], right, 0, right.length) <= 0) ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = + (StringExpr.compare(left, 0, left.length, vector[i], start[i], length[i]) <= 0 && + StringExpr.compare(vector[i], start[i], length[i], right, 0, right.length) <= 0) ? 1 : 0; + } + } + } else { + if (inputColVector.isRepeating) { + if (!nullPos[0]) { + outputVector[0] = + (StringExpr.compare(left, 0, left.length, vector[0], start[0], length[0]) <= 0 && + StringExpr.compare(vector[0], start[0], length[0], right, 0, right.length) <= 0) ? 1 : 0; + } + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + outputVector[i] = + (StringExpr.compare(left, 0, left.length, vector[i], start[i], length[i]) <= 0 && + StringExpr.compare(vector[i], start[i], length[i], right, 0, right.length) <= 0) ? 1 : 0; + } + } + } else { + for(int i = 0; i != n; i++) { + if (!nullPos[i]) { + outputVector[i] = + (StringExpr.compare(left, 0, left.length, vector[i], start[i], length[i]) <= 0 && + StringExpr.compare(vector[i], start[i], length[i], right, 0, right.length) <= 0) ? 1 : 0; + } + } + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public byte[] getLeftValue() { + return left; + } + + public void setLeftValue(byte[] value) { + this.left = value; + } + + public byte[] getRightValue() { + return right; + } + + public void setRightValue(byte[] value) { + this.right = value; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + // This VectorExpression (BETWEEN) is a special case, so don't return a descriptor. + return null; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 298f429..6bdd24b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.exec.TaskResult; import org.apache.hadoop.hive.ql.exec.TaskRunner; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.history.HiveHistory; import org.apache.hadoop.hive.ql.history.HiveHistory.Keys; import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; @@ -1280,20 +1281,21 @@ public int execute() throws CommandNeedRetryException { maxthreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.EXECPARALLETHREADNUMBER); + SessionState ss = SessionState.get(); + HiveHistory history = ss.getHiveHistory(); + try { LOG.info("Starting command: " + queryStr); plan.setStarted(); - if (SessionState.get() != null) { - SessionState.get().getHiveHistory().startQuery(queryStr, - conf.getVar(HiveConf.ConfVars.HIVEQUERYID)); - SessionState.get().getHiveHistory().logPlanProgress(plan); - } + history.startQuery(queryStr, conf.getVar(HiveConf.ConfVars.HIVEQUERYID)); + history.logPlanProgress(plan); + resStream = null; - SessionState ss = SessionState.get(); - HookContext hookContext = new HookContext(plan, conf, ctx.getPathToCS(), ss.getUserName(), ss.getUserIpAddress()); + HookContext hookContext = new HookContext(plan, conf, ctx.getPathToCS(), + ss.getUserName(), ss.getUserIpAddress()); hookContext.setHookType(HookContext.HookType.PRE_EXEC_HOOK); for (Hook peh : getHooks(HiveConf.ConfVars.PREEXECHOOKS)) { @@ -1306,7 +1308,7 @@ public int execute() throws CommandNeedRetryException { } else if (peh instanceof PreExecute) { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PRE_HOOK + peh.getClass().getName()); - ((PreExecute) peh).run(SessionState.get(), plan.getInputs(), plan.getOutputs(), + ((PreExecute) peh).run(ss, plan.getInputs(), plan.getOutputs(), ShimLoader.getHadoopShims().getUGIForConf(conf)); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PRE_HOOK + peh.getClass().getName()); @@ -1319,11 +1321,9 @@ public int execute() throws CommandNeedRetryException { console.printInfo("Query ID = " + plan.getQueryId()); console.printInfo("Total jobs = " + jobs); } - if (SessionState.get() != null) { - SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_NUM_TASKS, - String.valueOf(jobs)); - SessionState.get().getHiveHistory().setIdToTableMap(plan.getIdToTableNameMap()); - } + history.setQueryProperty(queryId, Keys.QUERY_NUM_TASKS, String.valueOf(jobs)); + history.setIdToTableMap(plan.getIdToTableNameMap()); + String jobname = Utilities.abbreviate(queryStr, maxlen - 6); // A runtime that launches runnable tasks as separate Threads through @@ -1339,10 +1339,7 @@ public int execute() throws CommandNeedRetryException { this.driverCxt = driverCxt; // for canceling the query (should be bound to session?) - SessionState.get().setMapRedStats(new LinkedHashMap()); - SessionState.get().setStackTraces(new HashMap>>()); - SessionState.get().setLocalMapRedErrors(new HashMap>()); - + ss.resetStates(); // Add root Tasks to runnable for (Task tsk : plan.getRootTasks()) { // This should never happen, if it does, it's a bug with the potential to produce @@ -1421,11 +1418,9 @@ public int execute() throws CommandNeedRetryException { driverCxt.finished(tskRun); - if (SessionState.get() != null) { - SessionState.get().getHiveHistory().setTaskProperty(queryId, tsk.getId(), - Keys.TASK_RET_CODE, String.valueOf(exitVal)); - SessionState.get().getHiveHistory().endTask(queryId, tsk); - } + history.setTaskProperty(queryId, tsk.getId(), + Keys.TASK_RET_CODE, String.valueOf(exitVal)); + history.endTask(queryId, tsk); if (tsk.getChildTasks() != null) { for (Task child : tsk.getChildTasks()) { @@ -1474,28 +1469,24 @@ public int execute() throws CommandNeedRetryException { } else if (peh instanceof PostExecute) { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.POST_HOOK + peh.getClass().getName()); - ((PostExecute) peh).run(SessionState.get(), plan.getInputs(), plan.getOutputs(), - (SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo() - : null), ShimLoader.getHadoopShims().getUGIForConf(conf)); + ((PostExecute) peh).run(ss, plan.getInputs(), plan.getOutputs(), + ss.getLineageState().getLineageInfo(), + ShimLoader.getHadoopShims().getUGIForConf(conf)); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.POST_HOOK + peh.getClass().getName()); } } + history.setQueryProperty(queryId, Keys.QUERY_RET_CODE, + String.valueOf(0)); + history.printRowCount(queryId); - if (SessionState.get() != null) { - SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_RET_CODE, - String.valueOf(0)); - SessionState.get().getHiveHistory().printRowCount(queryId); - } } catch (CommandNeedRetryException e) { throw e; } catch (Exception e) { ctx.restoreOriginalTracker(); - if (SessionState.get() != null) { - SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_RET_CODE, - String.valueOf(12)); - } + history.setQueryProperty(queryId, Keys.QUERY_RET_CODE, String.valueOf(12)); + // TODO: do better with handling types of Exception here errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e); SQLState = "08S01"; @@ -1504,15 +1495,13 @@ public int execute() throws CommandNeedRetryException { + org.apache.hadoop.util.StringUtils.stringifyException(e)); return (12); } finally { - if (SessionState.get() != null) { - SessionState.get().getHiveHistory().endQuery(queryId); - } + history.endQuery(queryId); if (noName) { conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, ""); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_EXECUTE); - Map stats = SessionState.get().getMapRedStats(); + Map stats = ss.getMapRedStats(); if (stats != null && !stats.isEmpty()) { long totalCpu = 0; console.printInfo("MapReduce Jobs Launched: "); @@ -1525,13 +1514,11 @@ public int execute() throws CommandNeedRetryException { } plan.setDone(); - if (SessionState.get() != null) { - try { - SessionState.get().getLineageState().clear(); - SessionState.get().getHiveHistory().logPlanProgress(plan); - } catch (Exception e) { - // ignore - } + try { + ss.getLineageState().clear(); + history.logPlanProgress(plan); + } catch (Exception e) { + // ignore } console.printInfo("OK"); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index acee377..99a9471 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.InputExpressionType; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Mode; import org.apache.hadoop.hive.ql.exec.vector.expressions.*; -import org.apache.hadoop.hive.ql.exec.vector.AggregateDefinition; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFAvgDecimal; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCount; @@ -831,7 +830,7 @@ private VectorExpression getIdentityExpression(List childExprList) } VectorExpression expr = new IdentityExpression(inputCol, colType); if (v1 != null) { - expr.setChildExpressions(new VectorExpression [] {v1}); + expr.setChildExpressions(v1); } return expr; } @@ -908,10 +907,12 @@ private VectorExpression createVectorExpression(Class vectorClass, throw new HiveException("Cannot handle expression type: " + child.getClass().getSimpleName()); } } - VectorExpression vectorExpression = instantiateExpression(vectorClass, returnType, arguments); - vectorExpression.setInputTypes(inputTypes); - if ((vectorExpression != null) && !children.isEmpty()) { - vectorExpression.setChildExpressions(children.toArray(new VectorExpression[0])); + VectorExpression vectorExpression = instantiateExpression(vectorClass, returnType, arguments); + if (vectorExpression != null) { + vectorExpression.setInputTypes(inputTypes); + if (!children.isEmpty()) { + vectorExpression.setChildExpressions(children.toArray(new VectorExpression[0])); + } } return vectorExpression; } catch (Exception ex) { @@ -1469,7 +1470,7 @@ private VectorExpression getCastToBoolean(List childExpr) int outputCol = ocm.allocateOutputColumn("Long"); VectorExpression lenToBoolExpr = new CastLongToBooleanViaLongToLong(lenExpr.getOutputColumn(), outputCol); - lenToBoolExpr.setChildExpressions(new VectorExpression[] {lenExpr}); + lenToBoolExpr.setChildExpressions(lenExpr); ocm.freeOutputColumn(lenExpr.getOutputColumn()); return lenToBoolExpr; } @@ -1510,43 +1511,34 @@ private VectorExpression getCastToLongExpression(List childExpr) private VectorExpression getBetweenFilterExpression(List childExpr, Mode mode, TypeInfo returnType) throws HiveException { - if (mode == Mode.PROJECTION) { - - // Projection mode is not yet supported for [NOT] BETWEEN. Return null so Vectorizer - // knows to revert to row-at-a-time execution. - return null; - } - - boolean notKeywordPresent = (Boolean) ((ExprNodeConstantDesc) childExpr.get(0)).getValue(); - ExprNodeDesc colExpr = childExpr.get(1); + ExprNodeDesc colExpr = childExpr.get(0); // The children after not, might need a cast. Get common types for the two comparisons. // Casting for 'between' is handled here as a special case, because the first child is for NOT and doesn't need // cast - TypeInfo commonType = FunctionRegistry.getCommonClassForComparison(childExpr.get(1).getTypeInfo(), - childExpr.get(2).getTypeInfo()); + TypeInfo commonType = FunctionRegistry.getCommonClassForComparison(colExpr.getTypeInfo(), + childExpr.get(1).getTypeInfo()); if (commonType == null) { // Can't vectorize return null; } - commonType = FunctionRegistry.getCommonClassForComparison(commonType, childExpr.get(3).getTypeInfo()); + commonType = FunctionRegistry.getCommonClassForComparison(commonType, childExpr.get(2).getTypeInfo()); if (commonType == null) { // Can't vectorize return null; } - List castChildren = new ArrayList(); + List castChildren = new ArrayList(childExpr.size()); - for (ExprNodeDesc desc: childExpr.subList(1, 4)) { + for (ExprNodeDesc desc: childExpr) { if (commonType.equals(desc.getTypeInfo())) { castChildren.add(desc); } else { GenericUDF castUdf = getGenericUDFForCast(commonType); - ExprNodeGenericFuncDesc engfd = new ExprNodeGenericFuncDesc(commonType, castUdf, - Arrays.asList(new ExprNodeDesc[] { desc })); - castChildren.add(engfd); + castChildren.add( + new ExprNodeGenericFuncDesc(commonType, castUdf, Arrays.asList(desc))); } } String colType = commonType.getTypeName(); @@ -1556,48 +1548,30 @@ private VectorExpression getBetweenFilterExpression(List childExpr // determine class Class cl = null; - if (isIntFamily(colType) && !notKeywordPresent) { - cl = FilterLongColumnBetween.class; - } else if (isIntFamily(colType) && notKeywordPresent) { - cl = FilterLongColumnNotBetween.class; - } else if (isFloatFamily(colType) && !notKeywordPresent) { - cl = FilterDoubleColumnBetween.class; - } else if (isFloatFamily(colType) && notKeywordPresent) { - cl = FilterDoubleColumnNotBetween.class; - } else if (colType.equals("string") && !notKeywordPresent) { - cl = FilterStringColumnBetween.class; - } else if (colType.equals("string") && notKeywordPresent) { - cl = FilterStringColumnNotBetween.class; - } else if (varcharTypePattern.matcher(colType).matches() && !notKeywordPresent) { - cl = FilterVarCharColumnBetween.class; - } else if (varcharTypePattern.matcher(colType).matches() && notKeywordPresent) { - cl = FilterVarCharColumnNotBetween.class; - } else if (charTypePattern.matcher(colType).matches() && !notKeywordPresent) { - cl = FilterCharColumnBetween.class; - } else if (charTypePattern.matcher(colType).matches() && notKeywordPresent) { - cl = FilterCharColumnNotBetween.class; + if (isIntFamily(colType)) { + cl = mode == Mode.FILTER ? FilterLongColumnBetween.class : LongColumnBetween.class; + } else if (isFloatFamily(colType)) { + cl = mode == Mode.FILTER ? FilterDoubleColumnBetween.class : DoubleColumnBetween.class; + } else if (colType.equals("string")) { + cl = mode == Mode.FILTER ? FilterStringColumnBetween.class : StringColumnBetween.class; + } else if (varcharTypePattern.matcher(colType).matches()) { + cl = mode == Mode.FILTER ? FilterVarCharColumnBetween.class : VarCharColumnBetween.class; + } else if (charTypePattern.matcher(colType).matches()) { + cl = mode == Mode.FILTER ? FilterCharColumnBetween.class : CharColumnBetween.class; } else if (colType.equals("timestamp")) { // Get timestamp boundary values as longs instead of the expected strings - long left = getTimestampScalar(childExpr.get(2)); - long right = getTimestampScalar(childExpr.get(3)); + long left = getTimestampScalar(childExpr.get(0)); + long right = getTimestampScalar(childExpr.get(1)); childrenAfterNot = new ArrayList(); childrenAfterNot.add(colExpr); childrenAfterNot.add(new ExprNodeConstantDesc(left)); childrenAfterNot.add(new ExprNodeConstantDesc(right)); - if (notKeywordPresent) { - cl = FilterLongColumnNotBetween.class; - } else { - cl = FilterLongColumnBetween.class; - } - } else if (isDecimalFamily(colType) && !notKeywordPresent) { - cl = FilterDecimalColumnBetween.class; - } else if (isDecimalFamily(colType) && notKeywordPresent) { - cl = FilterDecimalColumnNotBetween.class; - } else if (isDateFamily(colType) && !notKeywordPresent) { - cl = FilterLongColumnBetween.class; - } else if (isDateFamily(colType) && notKeywordPresent) { - cl = FilterLongColumnNotBetween.class; + cl = mode == Mode.FILTER ? FilterLongColumnBetween.class : LongColumnBetween.class; + } else if (isDecimalFamily(colType)) { + cl = mode == Mode.FILTER ? FilterDecimalColumnBetween.class : DecimalColumnBetween.class; + } else if (isDateFamily(colType)) { + cl = mode == Mode.FILTER ? FilterLongColumnBetween.class : LongColumnBetween.class; } return createVectorExpression(cl, childrenAfterNot, Mode.PROJECTION, returnType); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnBetween.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnBetween.java new file mode 100644 index 0000000..6070ae9 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnBetween.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.common.type.Decimal128; +import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +public class DecimalColumnBetween extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private int outputCol; + + // The comparison is of the form "column BETWEEN leftValue AND rightValue" + private Decimal128 leftValue; + private Decimal128 rightValue; + + public DecimalColumnBetween() { } + + public DecimalColumnBetween(int colNum, Decimal128 leftValue, Decimal128 rightValue, int outputCol) { + this.colNum = colNum; + this.leftValue = leftValue; + this.rightValue = rightValue; + this.outputCol = outputCol; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum]; + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputCol]; + + final int[] sel = batch.selected; + final boolean[] nullPos = inputColVector.isNull; + final boolean[] outNulls = outputColVector.isNull; + final int n = batch.size; + final Decimal128[] vector = inputColVector.vector; + final long[] outputVector = outputColVector.vector; + + outputColVector.isRepeating = false; + outputColVector.noNulls = inputColVector.noNulls; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.isRepeating) { + outNulls[0] = nullPos[0]; + outputColVector.isRepeating = true; + } else if (!inputColVector.noNulls) { + System.arraycopy(nullPos, 0, outNulls, 0, nullPos.length); + } + + if (inputColVector.noNulls) { + if (inputColVector.isRepeating) { + outputVector[0] = (leftValue.compareTo(vector[0]) <= 0 && vector[0].compareTo(rightValue) <= 0) ? 1 : 0; + } else if (batch.selectedInUse) { + for (int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = (leftValue.compareTo(vector[i]) <= 0 && vector[i].compareTo(rightValue) <= 0) ? 1 : 0; + } + } else { + for (int i = 0; i != n; i++) { + outputVector[i] = (leftValue.compareTo(vector[i]) <= 0 && vector[i].compareTo(rightValue) <= 0) ? 1 : 0; + } + } + } else { + if (inputColVector.isRepeating) { + if (!nullPos[0]) { + outputVector[0] = (leftValue.compareTo(vector[0]) <= 0 && vector[0].compareTo(rightValue) <= 0) ? 1 : 0; + } + } else if (batch.selectedInUse) { + for (int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + outputVector[i] = (leftValue.compareTo(vector[i]) <= 0 && vector[i].compareTo(rightValue) <= 0) ? 1 : 0; + } + } + } else { + for (int i = 0; i != n; i++) { + if (!nullPos[i]) { + outputVector[i] = (leftValue.compareTo(vector[i]) <= 0 && vector[i].compareTo(rightValue) <= 0) ? 1 : 0; + } + } + } + } + } + + @Override + public int getOutputColumn() { + return outputCol; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public Decimal128 getLeftValue() { + return leftValue; + } + + public void setLeftValue(Decimal128 value) { + this.leftValue = value; + } + + public Decimal128 getRightValue() { + return rightValue; + } + + public void setRightValue(Decimal128 value) { + this.rightValue = value; + } + + public void setOutputColumn(int outputCol) { + this.outputCol = outputCol; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return null; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnBetween.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnBetween.java new file mode 100644 index 0000000..30441bc --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnBetween.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +public class StringColumnBetween extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private int outputCol; + + // The comparison is of the form "column BETWEEN leftValue AND rightValue" + private byte[] left; + private byte[] right; + + public StringColumnBetween() { } + + public StringColumnBetween(int colNum, byte[] left, byte[] right, int outputCol) { + this.colNum = colNum; + this.left = left; + this.right = right; + this.outputCol = outputCol; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + final BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; + final LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputCol]; + + final int[] length = inputColVector.length; + final int[] start = inputColVector.start; + + final int[] sel = batch.selected; + final boolean[] nullPos = inputColVector.isNull; + final boolean[] outNulls = outputColVector.isNull; + final int n = batch.size; + final byte[][] vector = inputColVector.vector; + final long[] outputVector = outputColVector.vector; + + outputColVector.isRepeating = false; + outputColVector.noNulls = inputColVector.noNulls; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector.isRepeating) { + outNulls[0] = nullPos[0]; + outputColVector.isRepeating = true; + } else if (!inputColVector.noNulls) { + System.arraycopy(nullPos, 0, outNulls, 0, nullPos.length); + } + + if (inputColVector.noNulls) { + if (inputColVector.isRepeating) { + outputVector[0] = + (StringExpr.compare(left, 0, left.length, vector[0], start[0], length[0]) <= 0 && + StringExpr.compare(vector[0], start[0], length[0], right, 0, right.length) <= 0) ? 1 : 0; + } else if (batch.selectedInUse) { + for (int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = + (StringExpr.compare(left, 0, left.length, vector[i], start[i], length[i]) <= 0 && + StringExpr.compare(vector[i], start[i], length[i], right, 0, right.length) <= 0) ? 1 : 0; + } + } else { + for (int i = 0; i != n; i++) { + outputVector[i] = + (StringExpr.compare(left, 0, left.length, vector[i], start[i], length[i]) <= 0 && + StringExpr.compare(vector[i], start[i], length[i], right, 0, right.length) <= 0) ? 1 : 0; + } + } + } else { + if (inputColVector.isRepeating) { + if (!nullPos[0]) { + outputVector[0] = + (StringExpr.compare(left, 0, left.length, vector[0], start[0], length[0]) <= 0 && + StringExpr.compare(vector[0], start[0], length[0], right, 0, right.length) <= 0) ? 1 : 0; + } + } else if (batch.selectedInUse) { + for (int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + outputVector[i] = + (StringExpr.compare(left, 0, left.length, vector[i], start[i], length[i]) <= 0 && + StringExpr.compare(vector[i], start[i], length[i], right, 0, right.length) <= 0) ? 1 : 0; + } + } + } else { + for (int i = 0; i != n; i++) { + if (!nullPos[i]) { + outputVector[i] = + (StringExpr.compare(left, 0, left.length, vector[i], start[i], length[i]) <= 0 && + StringExpr.compare(vector[i], start[i], length[i], right, 0, right.length) <= 0) ? 1 : 0; + } + } + } + } + } + + @Override + public int getOutputColumn() { + return outputCol; + } + + @Override + public String getOutputType() { + return "boolean"; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + public byte[] getLeftValue() { + return left; + } + + public void setLeftValue(byte[] value) { + this.left = value; + } + + public byte[] getRightValue() { + return right; + } + + public void setRightValue(byte[] value) { + this.right = value; + } + + public void setOutputColumn(int outputCol) { + this.outputCol = outputCol; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return null; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java index d7ace6d..ffb63db 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpression.java @@ -97,7 +97,7 @@ public void setOutputType(String type) { /** * Initialize the child expressions. */ - public void setChildExpressions(VectorExpression [] ve) { + public void setChildExpressions(VectorExpression... ve) { childExpressions = ve; } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 34d2dfc..cafbc8f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -386,10 +386,10 @@ precedenceEqualExpression -> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_IN) subQueryExpression $precedenceEqualExpression) | (KW_IN expressions) -> ^(TOK_FUNCTION KW_IN $precedenceEqualExpression expressions) - | ( KW_NOT KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression) ) - -> ^(TOK_FUNCTION Identifier["between"] KW_TRUE $left $min $max) - | ( KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression) ) - -> ^(TOK_FUNCTION Identifier["between"] KW_FALSE $left $min $max) + | (KW_NOT KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression) ) + -> ^(KW_NOT ^(TOK_FUNCTION Identifier["between"] $left $min $max)) + | (KW_BETWEEN (min=precedenceBitwiseOrExpression) KW_AND (max=precedenceBitwiseOrExpression) ) + -> ^(TOK_FUNCTION Identifier["between"] $left $min $max) )* | (KW_EXISTS LPAREN KW_SELECT)=> (KW_EXISTS subQueryExpression) -> ^(TOK_SUBQUERY_EXPR ^(TOK_SUBQUERY_OP KW_EXISTS) subQueryExpression) ; diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index d7227ed..0c39535 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -25,7 +25,15 @@ import java.io.PrintStream; import java.net.URI; import java.net.URLClassLoader; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; @@ -154,12 +162,12 @@ private CreateTableAutomaticGrant createTableGrants; - private Map mapRedStats; + private Map mapRedStats = new LinkedHashMap(); private Map hiveVariables; // A mapping from a hadoop job ID to the stack traces collected from the map reduce task logs - private Map>> stackTraces; + private Map>> stackTraces = new HashMap>>(); // This mapping collects all the configuration variables which have been set by the user // explicitly, either via SET in the CLI, the hiveconf option, or a System property. @@ -167,7 +175,7 @@ // changes the value of a variable, the corresponding change will be made in this mapping. private Map overriddenConfigurations; - private Map> localMapRedErrors; + private Map> localMapRedErrors = new HashMap>(); private TezSessionState tezSessionState; @@ -565,6 +573,12 @@ public String getHdfsScratchDirURIString() { return hdfsScratchDirURIString; } + public void resetStates() { + mapRedStats.clear(); + stackTraces.clear(); + localMapRedErrors.clear(); + } + public static Path getLocalSessionPath(Configuration conf) { SessionState ss = SessionState.get(); if (ss == null) { diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBetween.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBetween.java index 9d9ee57..ddb8d5b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBetween.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBetween.java @@ -20,29 +20,24 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; -import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.BooleanWritable; @Description(name = "between", value = "_FUNC_ a [NOT] BETWEEN b AND c - evaluate if a is [not] in between b and c") public class GenericUDFBetween extends GenericUDF { - GenericUDFOPEqualOrGreaterThan egt = new GenericUDFOPEqualOrGreaterThan(); - GenericUDFOPEqualOrLessThan elt = new GenericUDFOPEqualOrLessThan(); + private GenericUDFOPEqualOrGreaterThan egt = new GenericUDFOPEqualOrGreaterThan(); + private GenericUDFOPEqualOrLessThan elt = new GenericUDFOPEqualOrLessThan(); private ObjectInspector[] argumentOIs; private final BooleanWritable result = new BooleanWritable(); @Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { - if (!arguments[0].getTypeName().equals("boolean")) { - throw new UDFArgumentTypeException(0, "First argument for BETWEEN should be boolean type"); - } - egt.initialize(new ObjectInspector[] {arguments[1], arguments[2]}); - elt.initialize(new ObjectInspector[] {arguments[1], arguments[3]}); + egt.initialize(new ObjectInspector[] {arguments[0], arguments[1]}); + elt.initialize(new ObjectInspector[] {arguments[0], arguments[2]}); argumentOIs = arguments; return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector; @@ -50,35 +45,28 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { - boolean invert = (Boolean) ((PrimitiveObjectInspector) argumentOIs[0]) - .getPrimitiveJavaObject(arguments[0].get()); - - BooleanWritable left = ((BooleanWritable)egt.evaluate(new DeferredObject[] {arguments[1], arguments[2]})); + BooleanWritable left = ((BooleanWritable)egt.evaluate(new DeferredObject[] {arguments[0], arguments[1]})); if (left == null) { return null; } - if (!invert && !left.get()) { + if (!left.get()) { result.set(false); return result; } - BooleanWritable right = ((BooleanWritable)elt.evaluate(new DeferredObject[] {arguments[1], arguments[3]})); + BooleanWritable right = ((BooleanWritable)elt.evaluate(new DeferredObject[] {arguments[0], arguments[2]})); if (right == null) { return null; } - boolean between = left.get() && right.get(); - result.set(invert ? !between : between); + result.set(right.get()); return result; } @Override public String getDisplayString(String[] children) { - StringBuilder sb = new StringBuilder(); - sb.append(children[1]); - if (Boolean.valueOf(children[0])) { - sb.append(" NOT"); - } + StringBuilder sb = new StringBuilder(32); + sb.append(children[0]); sb.append(" BETWEEN "); - sb.append(children[2]).append(" AND ").append(children[3]); + sb.append(children[1]).append(" AND ").append(children[2]); return sb.toString(); } } diff --git ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java index 93981fa..2992871 100644 --- ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java +++ ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java @@ -141,7 +141,7 @@ public void testPartitionExpr() throws Exception { checkExpr(1, dbName, tblName, e.val("p11").strCol("p1").pred(">", 2) .intCol("p2").val(31).pred("<", 2).pred("and", 2).build()); checkExpr(3, dbName, tblName, - e.val(32).val(31).intCol("p2").val(false).pred("between", 4).build()); + e.val(32).val(31).intCol("p2").pred("between", 3).build()); // Apply isnull and instr (not supported by pushdown) via name filtering. checkExpr(4, dbName, tblName, e.val("p").strCol("p1") diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index 0eb7c5a..5fbcd4a 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -471,6 +471,8 @@ private void executePlan() throws Exception { DriverContext dctx = new DriverContext (); mrtask.setWork(mr); mrtask.initialize(conf, null, dctx); + + SessionState.get().resetStates(); int exitVal = mrtask.execute(dctx); if (exitVal != 0) { diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java index 24a52a6..0822412 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java @@ -97,7 +97,7 @@ public void testBasicFilterOperator() throws HiveException { VectorExpression ve1 = new FilterLongColGreaterLongColumn(0,1); VectorExpression ve2 = new FilterLongColEqualDoubleScalar(2, 0); VectorExpression ve3 = new FilterExprAndExpr(); - ve3.setChildExpressions(new VectorExpression[] {ve1, ve2}); + ve3.setChildExpressions(ve1, ve2); vfo.setFilterCondition(ve3); FakeDataReader fdr = new FakeDataReader(1024*1, 3); @@ -125,7 +125,7 @@ public void testBasicFilterLargeData() throws HiveException { VectorExpression ve1 = new FilterLongColGreaterLongColumn(0,1); VectorExpression ve2 = new FilterLongColEqualDoubleScalar(2, 0); VectorExpression ve3 = new FilterExprAndExpr(); - ve3.setChildExpressions(new VectorExpression[] {ve1, ve2}); + ve3.setChildExpressions(ve1, ve2); vfo.setFilterCondition(ve3); FakeDataReader fdr = new FakeDataReader(16*1024*1024, 3); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java index 9e3628f..98fe86e 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java @@ -30,7 +30,6 @@ import junit.framework.Assert; import org.apache.hadoop.hive.common.type.HiveChar; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.vector.expressions.ColAndCol; import org.apache.hadoop.hive.ql.exec.vector.expressions.ColOrCol; @@ -79,21 +78,16 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DoubleColUnaryMinus; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColLessDoubleScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnBetween; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColEqualLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColGreaterLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColLessDoubleScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetween; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongScalarGreaterLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringGroupColGreaterStringGroupColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringGroupColGreaterStringScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetween; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterVarCharColumnBetween; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterVarCharColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterCharColumnBetween; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterCharColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncLnDoubleToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncRoundDoubleToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncSinDoubleToDouble; @@ -135,12 +129,10 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFPower; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFRound; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPlus; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.junit.Test; @@ -1019,7 +1011,6 @@ public void testBetweenFilters() throws HiveException { // string BETWEEN GenericUDFBetween udf = new GenericUDFBetween(); List children1 = new ArrayList(); - children1.add(new ExprNodeConstantDesc(new Boolean(false))); // no NOT keyword children1.add(col1Expr); children1.add(constDesc); children1.add(constDesc2); @@ -1033,11 +1024,6 @@ public void testBetweenFilters() throws HiveException { VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterStringColumnBetween); - // string NOT BETWEEN - children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); // has NOT keyword - ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); - assertTrue(ve instanceof FilterStringColumnNotBetween); - // CHAR tests CharTypeInfo charTypeInfo = new CharTypeInfo(10); col1Expr = new ExprNodeColumnDesc(charTypeInfo, "col1", "table", false); @@ -1047,7 +1033,6 @@ public void testBetweenFilters() throws HiveException { // CHAR BETWEEN udf = new GenericUDFBetween(); children1 = new ArrayList(); - children1.add(new ExprNodeConstantDesc(new Boolean(false))); // no NOT keyword children1.add(col1Expr); children1.add(constDesc); children1.add(constDesc2); @@ -1058,11 +1043,6 @@ public void testBetweenFilters() throws HiveException { ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterCharColumnBetween); - // CHAR NOT BETWEEN - children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); // has NOT keyword - ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); - assertTrue(ve instanceof FilterCharColumnNotBetween); - // VARCHAR tests VarcharTypeInfo varcharTypeInfo = new VarcharTypeInfo(10); col1Expr = new ExprNodeColumnDesc(varcharTypeInfo, "col1", "table", false); @@ -1072,7 +1052,6 @@ public void testBetweenFilters() throws HiveException { // VARCHAR BETWEEN udf = new GenericUDFBetween(); children1 = new ArrayList(); - children1.add(new ExprNodeConstantDesc(new Boolean(false))); // no NOT keyword children1.add(col1Expr); children1.add(constDesc); children1.add(constDesc2); @@ -1083,49 +1062,26 @@ public void testBetweenFilters() throws HiveException { ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterVarCharColumnBetween); - // VARCHAR NOT BETWEEN - children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); // has NOT keyword - ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); - assertTrue(ve instanceof FilterVarCharColumnNotBetween); - // long BETWEEN - children1.set(0, new ExprNodeConstantDesc(new Boolean(false))); - children1.set(1, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); - children1.set(2, new ExprNodeConstantDesc(10)); - children1.set(3, new ExprNodeConstantDesc(20)); + children1.set(0, new ExprNodeColumnDesc(Long.class, "col1", "table", false)); + children1.set(1, new ExprNodeConstantDesc(10)); + children1.set(2, new ExprNodeConstantDesc(20)); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterLongColumnBetween); - // long NOT BETWEEN - children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); - ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); - assertTrue(ve instanceof FilterLongColumnNotBetween); - // double BETWEEN - children1.set(0, new ExprNodeConstantDesc(new Boolean(false))); - children1.set(1, new ExprNodeColumnDesc(Double.class, "col1", "table", false)); - children1.set(2, new ExprNodeConstantDesc(10.0d)); - children1.set(3, new ExprNodeConstantDesc(20.0d)); + children1.set(0, new ExprNodeColumnDesc(Double.class, "col1", "table", false)); + children1.set(1, new ExprNodeConstantDesc(10.0d)); + children1.set(2, new ExprNodeConstantDesc(20.0d)); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertTrue(ve instanceof FilterDoubleColumnBetween); - // double NOT BETWEEN - children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); - ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); - assertTrue(ve instanceof FilterDoubleColumnNotBetween); - // timestamp BETWEEN - children1.set(0, new ExprNodeConstantDesc(new Boolean(false))); - children1.set(1, new ExprNodeColumnDesc(Timestamp.class, "col1", "table", false)); - children1.set(2, new ExprNodeConstantDesc("2013-11-05 00:00:00.000")); - children1.set(3, new ExprNodeConstantDesc("2013-11-06 00:00:00.000")); + children1.set(0, new ExprNodeColumnDesc(Timestamp.class, "col1", "table", false)); + children1.set(1, new ExprNodeConstantDesc("2013-11-05 00:00:00.000")); + children1.set(2, new ExprNodeConstantDesc("2013-11-06 00:00:00.000")); ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); assertEquals(FilterStringColumnBetween.class, ve.getClass()); - - // timestamp NOT BETWEEN - children1.set(0, new ExprNodeConstantDesc(new Boolean(true))); - ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER); - assertEquals(FilterStringColumnNotBetween.class, ve.getClass()); } // Test translation of both IN filters and boolean-valued IN expressions (non-filters). diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java index 92e5a06..54963c5 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java @@ -31,25 +31,24 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DoubleColumnBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColGreaterEqualDecimalColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColLessDecimalScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalScalarGreaterDecimalColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnBetween; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColEqualLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColGreaterLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColGreaterLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColLessLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnBetween; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongScalarGreaterLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongScalarLessLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetween; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetween; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColEqualDecimalScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColEqualDecimalColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalScalarEqualDecimalColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColAddLongScalar; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColumnBetween; import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil; import org.junit.Assert; import org.junit.Test; @@ -202,7 +201,7 @@ public void testFilterLongColLessLongColumn() { LongColAddLongScalar childExpr = new LongColAddLongScalar(0, 10, 2); - expr.setChildExpressions(new VectorExpression[] {childExpr}); + expr.setChildExpressions(childExpr); //Basic case lcv0.vector[0] = 10; @@ -440,7 +439,9 @@ public void testFilterLongNotBetween() { lcv0.vector[3] = 15; lcv0.vector[4] = 10; - VectorExpression expr = new FilterLongColumnNotBetween(0, 10, 20); + VectorExpression expr = new SelectColumnIsFalse(0); + VectorExpression between = new LongColumnBetween(0, 10, 20, 0); + expr.setChildExpressions(between); expr.evaluate(vrb); assertEquals(1, vrb.size); assertTrue(vrb.selectedInUse); @@ -479,6 +480,7 @@ public void testFilterDoubleNotBetween() { VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch( 5, 2, seed); vrb.cols[0] = new DoubleColumnVector(); + vrb.cols[1] = new LongColumnVector(); DoubleColumnVector dcv = (DoubleColumnVector) vrb.cols[0]; //Basic case @@ -488,7 +490,9 @@ public void testFilterDoubleNotBetween() { dcv.vector[3] = 15; dcv.vector[4] = 10; - VectorExpression expr = new FilterDoubleColumnNotBetween(0, 10, 20); + VectorExpression expr = new SelectColumnIsFalse(1); + VectorExpression between = new DoubleColumnBetween(0, 10, 20, 1); + expr.setChildExpressions(between); expr.evaluate(vrb); assertEquals(1, vrb.size); assertTrue(vrb.selectedInUse); @@ -569,6 +573,7 @@ public void testFilterStringNotBetween() { VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch( 3, 2, seed); vrb.cols[0] = new BytesColumnVector(); + vrb.cols[1] = new LongColumnVector(); BytesColumnVector bcv = (BytesColumnVector) vrb.cols[0]; bcv.initBuffer(); @@ -576,7 +581,9 @@ public void testFilterStringNotBetween() { bcv.setVal(1, b, 0, 1); bcv.setVal(2, c, 0, 1); - VectorExpression expr = new FilterStringColumnNotBetween(0, b, c); + VectorExpression expr = new SelectColumnIsFalse(1); + VectorExpression between = new StringColumnBetween(0, b, c, 1); + expr.setChildExpressions(between); expr.evaluate(vrb); assertEquals(1, vrb.size); @@ -628,8 +635,11 @@ public void testFilterTimestampNotBetween() { lcv0.vector[2] = TimestampUtils.getTimeNanoSec(ts2); vrb.size = 3; - VectorExpression expr1 = new FilterLongColumnNotBetween(0, startTS, endTS); - expr1.evaluate(vrb); + VectorExpression expr = new SelectColumnIsFalse(1); + VectorExpression between = new LongColumnBetween(0, startTS, endTS, 1); + expr.setChildExpressions(between); + expr.evaluate(vrb); + assertEquals(2, vrb.size); assertEquals(true, vrb.selectedInUse); assertEquals(0, vrb.selected[0]); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorLogicalExpressions.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorLogicalExpressions.java index 60e20a3..9d11754 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorLogicalExpressions.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorLogicalExpressions.java @@ -408,7 +408,7 @@ public void testFilterExprOrExpr() { SelectColumnIsFalse expr2 = new SelectColumnIsFalse(1); FilterExprOrExpr orExpr = new FilterExprOrExpr(); - orExpr.setChildExpressions(new VectorExpression[] {expr1, expr2}); + orExpr.setChildExpressions(expr1, expr2); orExpr.evaluate(batch1); orExpr.evaluate(batch2); @@ -448,7 +448,7 @@ public void testFilterExprOrExprWithBatchReuse() { SelectColumnIsFalse expr2 = new SelectColumnIsFalse(1); FilterExprOrExpr orExpr = new FilterExprOrExpr(); - orExpr.setChildExpressions(new VectorExpression[] {expr1, expr2}); + orExpr.setChildExpressions(expr1, expr2); orExpr.evaluate(batch1); @@ -482,7 +482,7 @@ public void testFilterExprOrExprWithSelectInUse() { SelectColumnIsFalse expr2 = new SelectColumnIsFalse(1); FilterExprOrExpr orExpr = new FilterExprOrExpr(); - orExpr.setChildExpressions(new VectorExpression[] {expr1, expr2}); + orExpr.setChildExpressions(expr1, expr2); // Evaluate batch1 so that temporary arrays in the expression // have residual values to interfere in later computation @@ -512,7 +512,7 @@ public void testFilterExprAndExpr() { SelectColumnIsFalse expr2 = new SelectColumnIsFalse(1); FilterExprAndExpr andExpr = new FilterExprAndExpr(); - andExpr.setChildExpressions(new VectorExpression[] {expr1, expr2}); + andExpr.setChildExpressions(expr1, expr2); andExpr.evaluate(batch1); diff --git ql/src/test/queries/clientpositive/vector_between_in.q ql/src/test/queries/clientpositive/vector_between_in.q index 1bc6611..61457af 100644 --- ql/src/test/queries/clientpositive/vector_between_in.q +++ ql/src/test/queries/clientpositive/vector_between_in.q @@ -33,3 +33,17 @@ SELECT cdate FROM decimal_date_test WHERE cdate NOT BETWEEN CAST("1968-05-01" AS SELECT cdecimal1 FROM decimal_date_test WHERE cdecimal1 BETWEEN -20 AND 45.9918918919 ORDER BY cdecimal1; SELECT COUNT(*) FROM decimal_date_test WHERE cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351; + + +---- + +EXPLAIN +SELECT cdate, + cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE), + cdate NOT BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE) +FROM (select distinct cdate from decimal_date_test WHERE cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-06-30" AS DATE)) X; + +SELECT cdate, + cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE), + cdate NOT BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE) +FROM (select distinct cdate from decimal_date_test WHERE cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-06-30" AS DATE)) X; diff --git ql/src/test/results/clientpositive/udf_between.q.out ql/src/test/results/clientpositive/udf_between.q.out index cb63d1c..7372081 100644 --- ql/src/test/results/clientpositive/udf_between.q.out +++ ql/src/test/results/clientpositive/udf_between.q.out @@ -79,7 +79,7 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key + 100) NOT BETWEEN 100 AND 200 (type: boolean) + predicate: (not (key + 100) BETWEEN 100 AND 200) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) diff --git ql/src/test/results/clientpositive/vector_between_in.q.out ql/src/test/results/clientpositive/vector_between_in.q.out index 631ac19..b0ebdbb 100644 --- ql/src/test/results/clientpositive/vector_between_in.q.out +++ ql/src/test/results/clientpositive/vector_between_in.q.out @@ -270,7 +270,7 @@ STAGE PLANS: alias: decimal_date_test Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: cdate NOT BETWEEN 1968-05-01 AND 1971-09-01 (type: boolean) + predicate: (not cdate BETWEEN 1968-05-01 AND 1971-09-01) (type: boolean) Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cdate (type: date) @@ -362,7 +362,7 @@ STAGE PLANS: alias: decimal_date_test Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351 (type: boolean) + predicate: (not cdecimal1 BETWEEN -2000 AND 4390.1351351351) (type: boolean) Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE @@ -682,3 +682,211 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_date_test #### A masked pattern was here #### 6172 +PREHOOK: query: ---- + +EXPLAIN +SELECT cdate, + cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE), + cdate NOT BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE) +FROM (select distinct cdate from decimal_date_test WHERE cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-06-30" AS DATE)) X +PREHOOK: type: QUERY +POSTHOOK: query: ---- + +EXPLAIN +SELECT cdate, + cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE), + cdate NOT BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE) +FROM (select distinct cdate from decimal_date_test WHERE cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-06-30" AS DATE)) X +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: decimal_date_test + Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: cdate BETWEEN 1969-01-01 AND 1969-06-30 (type: boolean) + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cdate (type: date) + outputColumnNames: cdate + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: cdate (type: date) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: date) + sort order: + + Map-reduce partition columns: _col0 (type: date) + Statistics: Num rows: 6144 Data size: 1233808 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: date) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 3072 Data size: 616904 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: date), _col0 BETWEEN 1969-01-01 AND 1969-03-31 (type: boolean), (not _col0 BETWEEN 1969-01-01 AND 1969-03-31) (type: boolean) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3072 Data size: 616904 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3072 Data size: 616904 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cdate, + cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE), + cdate NOT BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE) +FROM (select distinct cdate from decimal_date_test WHERE cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-06-30" AS DATE)) X +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_date_test +#### A masked pattern was here #### +POSTHOOK: query: SELECT cdate, + cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE), + cdate NOT BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-03-31" AS DATE) +FROM (select distinct cdate from decimal_date_test WHERE cdate BETWEEN CAST("1969-01-01" AS DATE) AND CAST("1969-06-30" AS DATE)) X +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_date_test +#### A masked pattern was here #### +1969-01-01 true false +1969-01-02 true false +1969-01-03 true false +1969-01-05 true false +1969-01-07 true false +1969-01-11 true false +1969-01-13 true false +1969-01-14 true false +1969-01-16 true false +1969-01-17 true false +1969-01-18 true false +1969-01-19 true false +1969-01-24 true false +1969-01-25 true false +1969-01-26 true false +1969-01-27 true false +1969-01-30 true false +1969-01-31 true false +1969-02-01 true false +1969-02-02 true false +1969-02-05 true false +1969-02-06 true false +1969-02-07 true false +1969-02-08 true false +1969-02-09 true false +1969-02-10 true false +1969-02-11 true false +1969-02-12 true false +1969-02-13 true false +1969-02-14 true false +1969-02-15 true false +1969-02-16 true false +1969-02-17 true false +1969-02-18 true false +1969-02-20 true false +1969-02-22 true false +1969-02-23 true false +1969-02-26 true false +1969-02-27 true false +1969-02-28 true false +1969-03-01 true false +1969-03-02 true false +1969-03-03 true false +1969-03-04 true false +1969-03-06 true false +1969-03-07 true false +1969-03-09 true false +1969-03-10 true false +1969-03-12 true false +1969-03-13 true false +1969-03-14 true false +1969-03-16 true false +1969-03-17 true false +1969-03-18 true false +1969-03-20 true false +1969-03-21 true false +1969-03-22 true false +1969-03-23 true false +1969-03-24 true false +1969-03-25 true false +1969-03-26 true false +1969-03-27 true false +1969-03-28 true false +1969-03-29 true false +1969-03-31 true false +1969-04-01 false true +1969-04-05 false true +1969-04-07 false true +1969-04-08 false true +1969-04-09 false true +1969-04-10 false true +1969-04-12 false true +1969-04-13 false true +1969-04-15 false true +1969-04-16 false true +1969-04-17 false true +1969-04-19 false true +1969-04-20 false true +1969-04-22 false true +1969-04-23 false true +1969-04-25 false true +1969-04-26 false true +1969-04-28 false true +1969-04-30 false true +1969-05-02 false true +1969-05-05 false true +1969-05-06 false true +1969-05-07 false true +1969-05-08 false true +1969-05-10 false true +1969-05-14 false true +1969-05-16 false true +1969-05-17 false true +1969-05-20 false true +1969-05-21 false true +1969-05-22 false true +1969-05-24 false true +1969-05-25 false true +1969-05-26 false true +1969-05-27 false true +1969-05-28 false true +1969-05-29 false true +1969-05-30 false true +1969-06-01 false true +1969-06-03 false true +1969-06-05 false true +1969-06-06 false true +1969-06-07 false true +1969-06-08 false true +1969-06-09 false true +1969-06-11 false true +1969-06-12 false true +1969-06-13 false true +1969-06-15 false true +1969-06-16 false true +1969-06-17 false true +1969-06-19 false true +1969-06-20 false true +1969-06-22 false true +1969-06-24 false true +1969-06-25 false true +1969-06-26 false true +1969-06-27 false true +1969-06-29 false true +1969-06-30 false true