diff --git common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
index 29dc06dca1a..169dbba36d2 100644
--- common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
+++ common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
@@ -29,6 +29,20 @@
public static final int MAX_CHAR_LENGTH = 255;
+ /**
+ * Creates a HiveChar for the value.
+ *
+ * If the value is not representable at char(length) returns null.
+ */
+ public static HiveChar create(String constValue, int length) {
+ HiveChar maxCharConst = new HiveChar(constValue, MAX_CHAR_LENGTH);
+ HiveChar charVal = new HiveChar(constValue, length);
+ if (maxCharConst.equals(charVal)) {
+ return charVal;
+ }
+ return null;
+ }
+
public HiveChar() {
}
@@ -43,6 +57,7 @@ public HiveChar(HiveChar hc, int len) {
/**
* Set char value, padding or truncating the value to the size of len parameter.
*/
+ @Override
public void setValue(String val, int len) {
super.setValue(HiveBaseChar.getPaddedValue(val, len), -1);
}
@@ -59,15 +74,18 @@ public String getPaddedValue() {
return value;
}
+ @Override
public int getCharacterLength() {
String strippedValue = getStrippedValue();
return strippedValue.codePointCount(0, strippedValue.length());
}
+ @Override
public String toString() {
return getPaddedValue();
}
+ @Override
public int compareTo(HiveChar rhs) {
if (rhs == this) {
return 0;
@@ -75,6 +93,7 @@ public int compareTo(HiveChar rhs) {
return this.getStrippedValue().compareTo(rhs.getStrippedValue());
}
+ @Override
public boolean equals(Object rhs) {
if (rhs == this) {
return true;
@@ -85,7 +104,9 @@ public boolean equals(Object rhs) {
return this.getStrippedValue().equals(((HiveChar) rhs).getStrippedValue());
}
+ @Override
public int hashCode() {
return getStrippedValue().hashCode();
}
+
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index e7d71595c7a..1a583115308 100644
--- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -18,13 +18,6 @@
package org.apache.hadoop.hive.ql;
-import java.io.FileNotFoundException;
-import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
@@ -35,6 +28,13 @@
import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.security.AccessControlException;
+import java.io.FileNotFoundException;
+import java.text.MessageFormat;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
/**
* List of all error messages.
* This list contains both compile time and run-time errors.
@@ -469,6 +469,7 @@
LOAD_DATA_LAUNCH_JOB_PARSE_ERROR(10416, "Encountered parse error while parsing rewritten load data into insert query"),
RESOURCE_PLAN_ALREADY_EXISTS(10417, "Resource plan {0} already exists", true),
RESOURCE_PLAN_NOT_EXISTS(10418, "Resource plan {0} does not exist", true),
+ INCOMPATIBLE_STRUCT(10419, "Incompatible structs.", true),
//========================== 20000 range starts here ========================//
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
index 04800cca91b..99cf7094d36 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
@@ -20,11 +20,13 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
@@ -34,11 +36,9 @@
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexUtil;
@@ -59,15 +59,18 @@
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
-public abstract class HivePointLookupOptimizerRule extends RelOptRule {
-
/**
- * This optimization will take a Filter or expression, and if its predicate contains
- * an OR operator whose children are constant equality expressions, it will try
- * to generate an IN clause (which is more efficient). If the OR operator contains
- * AND operator children, the optimization might generate an IN clause that uses
- * structs.
+ * This optimization attempts to identify and close expanded INs.
+ *
+ * Basically:
+ *
+ * (c) IN ( v1, v2, ...) <=> c1=v1 || c1=v2 || ...
+ *
+ * If c is struct; then c=v1 is a group of anded equations.
*/
+public abstract class HivePointLookupOptimizerRule extends RelOptRule {
+
+ /** Rule adapter to apply the transformation to Filter conditions. */
public static class FilterCondition extends HivePointLookupOptimizerRule {
public FilterCondition (int minNumORClauses) {
super(operand(Filter.class, any()), minNumORClauses);
@@ -87,13 +90,7 @@ public void onMatch(RelOptRuleCall call) {
}
}
-/**
- * This optimization will take a Join or expression, and if its join condition contains
- * an OR operator whose children are constant equality expressions, it will try
- * to generate an IN clause (which is more efficient). If the OR operator contains
- * AND operator children, the optimization might generate an IN clause that uses
- * structs.
- */
+ /** Rule adapter to apply the transformation to Join conditions. */
public static class JoinCondition extends HivePointLookupOptimizerRule {
public JoinCondition (int minNumORClauses) {
super(operand(Join.class, any()), minNumORClauses);
@@ -118,6 +115,42 @@ public void onMatch(RelOptRuleCall call) {
}
}
+ /** Rule adapter to apply the transformation to Projections. */
+ public static class Projections extends HivePointLookupOptimizerRule {
+ public Projections(int minNumORClauses) {
+ super(operand(Project.class, any()), minNumORClauses);
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final Project project = call.rel(0);
+ boolean changed = false;
+ final RexBuilder rexBuilder = project.getCluster().getRexBuilder();
+ List newProjects = new ArrayList<>();
+ for (RexNode oldNode : project.getProjects()) {
+ RexNode newNode = analyzeRexNode(rexBuilder, oldNode);
+ if (!newNode.toString().equals(oldNode.toString())) {
+ changed = true;
+ newProjects.add(newNode);
+ } else {
+ newProjects.add(oldNode);
+ }
+ }
+ if (!changed) {
+ return;
+ }
+ Project newProject = project.copy(project.getTraitSet(), project.getInput(), newProjects,
+ project.getRowType(), project.getFlags());
+ call.transformTo(newProject);
+
+ }
+
+ @Override
+ protected RelNode copyNode(AbstractRelNode node, RexNode newCondition) {
+ return null;
+ }
+ }
+
protected static final Logger LOG = LoggerFactory.getLogger(HivePointLookupOptimizerRule.class);
// Minimum number of OR clauses needed to transform into IN clauses
@@ -136,14 +169,7 @@ public void analyzeCondition(RelOptRuleCall call,
AbstractRelNode node,
RexNode condition) {
- // 1. We try to transform possible candidates
- RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, node,
- minNumORClauses);
- RexNode newCondition = transformIntoInClause.apply(condition);
-
- // 2. We merge IN expressions
- RexMergeInClause mergeInClause = new RexMergeInClause(rexBuilder);
- newCondition = mergeInClause.apply(newCondition);
+ RexNode newCondition = analyzeRexNode(rexBuilder, condition);
// 3. If we could not transform anything, we bail out
if (newCondition.toString().equals(condition.toString())) {
@@ -156,17 +182,26 @@ public void analyzeCondition(RelOptRuleCall call,
call.transformTo(newNode);
}
+ public RexNode analyzeRexNode(RexBuilder rexBuilder, RexNode condition) {
+ // 1. We try to transform possible candidates
+ RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, minNumORClauses);
+ RexNode newCondition = transformIntoInClause.apply(condition);
+
+ // 2. We merge IN expressions
+ RexMergeInClause mergeInClause = new RexMergeInClause(rexBuilder);
+ newCondition = mergeInClause.apply(newCondition);
+ return newCondition;
+ }
+
/**
* Transforms OR clauses into IN clauses, when possible.
*/
protected static class RexTransformIntoInClause extends RexShuttle {
private final RexBuilder rexBuilder;
- private final AbstractRelNode nodeOp;
private final int minNumORClauses;
- RexTransformIntoInClause(RexBuilder rexBuilder, AbstractRelNode nodeOp, int minNumORClauses) {
- this.nodeOp = nodeOp;
+ RexTransformIntoInClause(RexBuilder rexBuilder, int minNumORClauses) {
this.rexBuilder = rexBuilder;
this.minNumORClauses = minNumORClauses;
}
@@ -180,7 +215,7 @@ public RexNode visitCall(RexCall inputCall) {
case OR:
try {
RexNode newNode = transformIntoInClauseCondition(rexBuilder,
- nodeOp.getRowType(), call, minNumORClauses);
+ call, minNumORClauses);
if (newNode != null) {
return newNode;
}
@@ -196,18 +231,60 @@ public RexNode visitCall(RexCall inputCall) {
}
/**
- * Represents a simple contraint.
+ * This class just wraps around a RexNode enables equals/hashCode based on toString.
+ *
+ * After CALCITE-2632 this might not be needed anymore */
+ static class RexNodeRef {
+
+ public static Comparator COMPARATOR = new Comparator() {
+ @Override
+ public int compare(RexNodeRef o1, RexNodeRef o2) {
+ return o1.node.toString().compareTo(o2.node.toString());
+ }
+ };
+ private RexNode node;
+
+ public RexNodeRef(RexNode node) {
+ this.node = node;
+ }
+
+ public RexNode getRexNode() {
+ return node;
+ }
+
+ @Override
+ public int hashCode() {
+ return node.toString().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof RexNodeRef) {
+ RexNodeRef otherRef = (RexNodeRef) o;
+ return node.toString().equals(otherRef.node.toString());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "ref for:" + node.toString();
+ }
+ }
+ /**
+ * Represents a contraint.
*
* Example: a=1
+ * substr(a,1,2) = concat('asd','xxx')
*/
static class Constraint {
- private RexLiteral literal;
- private RexInputRef inputRef;
+ private RexNode literal;
+ private RexNode inputRef;
- public Constraint(RexInputRef inputRef, RexLiteral literal) {
- this.literal = literal;
- this.inputRef = inputRef;
+ public Constraint(RexNode opB, RexNode opA) {
+ this.literal = opA;
+ this.inputRef = opB;
}
/**
@@ -223,21 +300,31 @@ public static Constraint of(RexNode n) {
}
RexNode opA = call.operands.get(0);
RexNode opB = call.operands.get(1);
- if (opA instanceof RexLiteral && opB instanceof RexInputRef) {
- RexLiteral rexLiteral = (RexLiteral) opA;
- RexInputRef rexInputRef = (RexInputRef) opB;
- return new Constraint(rexInputRef, rexLiteral);
+ if (RexUtil.isNull(opA) || RexUtil.isNull(opB)) {
+ // dont try to compare nulls
+ return null;
+ }
+ if (isConstExpr(opA) && isColumnExpr(opB)) {
+ return new Constraint(opB, opA);
}
- if (opA instanceof RexInputRef && opB instanceof RexLiteral) {
- RexLiteral rexLiteral = (RexLiteral) opB;
- RexInputRef rexInputRef = (RexInputRef) opA;
- return new Constraint(rexInputRef, rexLiteral);
+ if (isColumnExpr(opA) && isConstExpr(opB)) {
+ return new Constraint(opA, opB);
}
return null;
}
- public RexInputRef getKey() {
- return inputRef;
+ private static boolean isColumnExpr(RexNode node) {
+ return !node.getType().isStruct() && HiveCalciteUtil.getInputRefs(node).size() > 0
+ && HiveCalciteUtil.isDeterministic(node);
+ }
+
+ private static boolean isConstExpr(RexNode node) {
+ return !node.getType().isStruct() && HiveCalciteUtil.getInputRefs(node).size() == 0
+ && HiveCalciteUtil.isDeterministic(node);
+ }
+
+ public RexNodeRef getKey() {
+ return new RexNodeRef(inputRef);
}
}
@@ -255,16 +342,17 @@ public RexInputRef getKey() {
*/
static class ConstraintGroup {
- public static final Function> KEY_FUNCTION = new Function>() {
+ public static final Function> KEY_FUNCTION =
+ new Function>() {
@Override
- public Set apply(ConstraintGroup a) {
+ public Set apply(ConstraintGroup a) {
return a.key;
}
};
- private Map constraints = new HashMap<>();
+ private Map constraints = new HashMap<>();
private RexNode originalRexNode;
- private final Set key;
+ private final Set key;
public ConstraintGroup(RexNode rexNode) {
originalRexNode = rexNode;
@@ -289,9 +377,9 @@ public ConstraintGroup(RexNode rexNode) {
key = constraints.keySet();
}
- public List getValuesInOrder(List columns) throws SemanticException {
+ public List getValuesInOrder(List columns) throws SemanticException {
List ret = new ArrayList<>();
- for (RexInputRef rexInputRef : columns) {
+ for (RexNodeRef rexInputRef : columns) {
Constraint constraint = constraints.get(rexInputRef);
if (constraint == null) {
throw new SemanticException("Unable to find constraint which was earlier added.");
@@ -302,8 +390,8 @@ public ConstraintGroup(RexNode rexNode) {
}
}
- private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RelDataType inputSchema,
- RexNode condition, int minNumORClauses) throws SemanticException {
+ private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RexNode condition,
+ int minNumORClauses) throws SemanticException {
assert condition.getKind() == SqlKind.OR;
ImmutableList operands = RexUtil.flattenOr(((RexCall) condition).getOperands());
@@ -318,10 +406,10 @@ private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RelDataTyp
allNodes.add(m);
}
- Multimap, ConstraintGroup> assignmentGroups =
+ Multimap, ConstraintGroup> assignmentGroups =
Multimaps.index(allNodes, ConstraintGroup.KEY_FUNCTION);
- for (Entry, Collection> sa : assignmentGroups.asMap().entrySet()) {
+ for (Entry, Collection> sa : assignmentGroups.asMap().entrySet()) {
// skip opaque
if (sa.getKey().size() == 0) {
continue;
@@ -351,22 +439,28 @@ private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RelDataTyp
}
- private RexNode buildInFor(Set set, Collection value) throws SemanticException {
+ private RexNode buildInFor(Set set, Collection value) throws SemanticException {
- List columns = new ArrayList();
+ List columns = new ArrayList<>();
columns.addAll(set);
+ columns.sort(RexNodeRef.COMPARATOR);
Listoperands = new ArrayList<>();
operands.add(useStructIfNeeded(columns));
for (ConstraintGroup node : value) {
List values = node.getValuesInOrder(columns);
- operands.add(useStructIfNeeded(values));
+ operands.add(useStructIfNeeded2(values));
}
return rexBuilder.makeCall(HiveIn.INSTANCE, operands);
}
- private RexNode useStructIfNeeded(List extends RexNode> columns) {
+ private RexNode useStructIfNeeded(List columns) {
+ return useStructIfNeeded2(columns.stream().map(n -> n.getRexNode()).collect(Collectors.toList()));
+
+ }
+
+ private RexNode useStructIfNeeded2(List extends RexNode> columns) {
// Create STRUCT clause
if (columns.size() == 1) {
return columns.get(0);
diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 82e975a50de..18a03591796 100644
--- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -25,6 +25,7 @@
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
+
import org.antlr.runtime.ClassicToken;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.tree.Tree;
@@ -277,7 +278,6 @@
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.joda.time.Interval;
-import javax.sql.DataSource;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
@@ -304,6 +304,8 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import javax.sql.DataSource;
+
public class CalcitePlanner extends SemanticAnalyzer {
@@ -2064,6 +2066,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv
if (conf.getBoolVar(HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZER)) {
rules.add(new HivePointLookupOptimizerRule.FilterCondition(minNumORClauses));
rules.add(new HivePointLookupOptimizerRule.JoinCondition(minNumORClauses));
+ rules.add(new HivePointLookupOptimizerRule.Projections(minNumORClauses));
}
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_CONSTRAINTS_JOIN) &&
profilesCBO.contains(ExtendedCBOProfile.REFERENTIAL_CONSTRAINTS)) {
diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
index 1b56ecd0440..12e7ae62f45 100644
--- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
@@ -19,9 +19,10 @@
package org.apache.hadoop.hive.ql.parse;
import org.apache.calcite.rel.RelNode;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+
import java.util.Map;
/**
@@ -263,4 +264,8 @@ public boolean isUseCaching() {
public boolean isFoldExpr() {
return foldExpr;
}
+
+ public boolean isCBOExecuted() {
+ return foldExpr;
+ }
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
index 4968d16876c..3833a278911 100644
--- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
@@ -75,6 +75,7 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFNvl;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
@@ -353,14 +354,18 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx,
}
public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull) {
- // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
- // precision and scale of the value by throwing away trailing zeroes. This may or may
- // not be desirable for the literals; however, this used to be the default behavior
- // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
HiveDecimal hd = HiveDecimal.create(strVal);
if (notNull && hd == null) {
return null;
}
+ return new ExprNodeConstantDesc(adjustType(hd), hd);
+ }
+
+ private static DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
int prec = 1;
int scale = 0;
if (hd != null) {
@@ -368,7 +373,7 @@ public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull)
scale = hd.scale();
}
DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
- return new ExprNodeConstantDesc(typeInfo, hd);
+ return typeInfo;
}
}
@@ -1163,36 +1168,59 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr,
ExprNodeDesc constChild = children.get(constIdx);
ExprNodeDesc columnChild = children.get(1 - constIdx);
- final PrimitiveTypeInfo colTypeInfo =
- TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase());
- ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild);
- if (newChild == null) {
- // non-interpretabe as that type...
- if (genericUDF instanceof GenericUDFOPEqual) {
- return new ExprNodeConstantDesc(false);
- }
- } else {
- children.set(constIdx, newChild);
+ final PrimitiveTypeInfo colTypeInfo =
+ TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase());
+ ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild);
+ if (newChild == null) {
+ // non-interpretable as target type...
+ // TODO: all comparisons with null should result in null
+ if (genericUDF instanceof GenericUDFOPEqual
+ && !(genericUDF instanceof GenericUDFOPEqualNS)) {
+ return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, null);
}
+ } else {
+ children.set(constIdx, newChild);
+ }
}
- if (genericUDF instanceof GenericUDFIn && children.get(0) instanceof ExprNodeColumnDesc) {
- ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) children.get(0);
- final PrimitiveTypeInfo colTypeInfo =
- TypeInfoFactory.getPrimitiveTypeInfo(columnDesc.getTypeString().toLowerCase());
+ if (genericUDF instanceof GenericUDFIn) {
+
+ ExprNodeDesc columnDesc = children.get(0);
List outputOpList = children.subList(1, children.size());
ArrayList inOperands = new ArrayList<>(outputOpList);
outputOpList.clear();
+ boolean hasNullValue = false;
for (ExprNodeDesc oldChild : inOperands) {
- if(oldChild !=null && oldChild instanceof ExprNodeConstantDesc) {
- ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, oldChild);
- if(newChild == null) {
- // non interpretable as target type; skip
- continue;
+ if (oldChild == null) {
+ hasNullValue = true;
+ continue;
+ }
+ ExprNodeDesc newChild = interpretNodeAsStruct(columnDesc, oldChild);
+ if (newChild == null) {
+ hasNullValue = true;
+ continue;
+ }
+ outputOpList.add(newChild);
+ }
+
+ if (hasNullValue) {
+ ExprNodeConstantDesc nullConst = new ExprNodeConstantDesc(columnDesc.getTypeInfo(), null);
+ if (outputOpList.size() == 0) {
+ // we have found only null values...remove the IN ; it will be null all the time.
+ return nullConst;
+ }
+ outputOpList.add(nullConst);
+ }
+ if (!ctx.isCBOExecuted()) {
+ ArrayList orOperands = rewriteInToOR(children);
+ if (orOperands != null) {
+ if (orOperands.size() == 1) {
+ orOperands.add(new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, false));
}
- outputOpList.add(newChild);
- }else{
- outputOpList.add(oldChild);
+ funcText = "or";
+ genericUDF = new GenericUDFOPOr();
+ children.clear();
+ children.addAll(orOperands);
}
}
}
@@ -1258,48 +1286,244 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr,
return desc;
}
- private ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) {
+ private ArrayList rewriteInToOR(ArrayList inOperands)
+ throws SemanticException {
+ ExprNodeDesc columnDesc = inOperands.get(0);
+
+ ArrayList orOperands = new ArrayList<>();
+ for (int i = 1; i < inOperands.size(); i++) {
+ ExprNodeDesc andExpr = buildEqualsArr(columnDesc, inOperands.get(i));
+ if (andExpr == null) {
+ return null;
+ }
+ orOperands.add(andExpr);
+ }
+ return orOperands;
+ }
+
+ private ExprNodeDesc buildEqualsArr(ExprNodeDesc columnDesc, ExprNodeDesc exprNodeDesc) throws SemanticException {
+ List lNodes = asListOfNodes(columnDesc);
+ List rNodes = asListOfNodes(exprNodeDesc);
+ if (lNodes == null || rNodes == null) {
+ // something went wrong
+ return null;
+ }
+ if(lNodes.size()!=rNodes.size()) {
+ throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnDesc + " and " + exprNodeDesc));
+ }
+
+ List ret = new ArrayList<>();
+ for (int i = 0; i < lNodes.size(); i++) {
+ ret.add(buildEquals(lNodes.get(i), rNodes.get(i)));
+ }
+ return buildAnd(ret);
+ }
+
+ private ExprNodeGenericFuncDesc buildEquals(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc) {
+ return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPEqual(),
+ Lists.newArrayList(columnDesc, valueDesc));
+ }
+
+ private ExprNodeDesc buildAnd(List values) {
+ if (values.size() == 1) {
+ return values.get(0);
+ } else {
+ return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(),
+ values);
+ }
+ }
+
+ private ExprNodeGenericFuncDesc buildOr(List values) {
+ return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPOr(),
+ values);
+ }
+
+ private List asListOfNodes(ExprNodeDesc desc) {
+ ExprNodeDesc valueDesc = desc;
+ if (ExprNodeDescUtils.isStructUDF(desc)) {
+ List valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren();
+ for (ExprNodeDesc exprNodeDesc : valueChilds) {
+ if (!isSafeExpression(exprNodeDesc)) {
+ return null;
+ }
+ }
+ return valueChilds;
+ }
+ if (ExprNodeDescUtils.isConstantStruct(valueDesc)) {
+ ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc;
+ List