();
@@ -175,11 +178,19 @@ private ExprNodeDesc analyzeExpr(
ExprNodeDesc expr1 = (ExprNodeDesc) nodeOutputs[0];
ExprNodeDesc expr2 = (ExprNodeDesc) nodeOutputs[1];
ExprNodeDesc[] extracted = ExprNodeDescUtils.extractComparePair(expr1, expr2);
- if (extracted == null) {
+ if (extracted == null || (extracted.length > 2 && !acceptsFields)) {
return expr;
}
- if (extracted.length > 2) {
+
+ ExprNodeColumnDesc columnDesc;
+ ExprNodeConstantDesc constantDesc;
+ if (extracted[0] instanceof ExprNodeConstantDesc) {
genericUDF = genericUDF.flip();
+ columnDesc = (ExprNodeColumnDesc) extracted[1];
+ constantDesc = (ExprNodeConstantDesc) extracted[0];
+ } else {
+ columnDesc = (ExprNodeColumnDesc) extracted[0];
+ constantDesc = (ExprNodeConstantDesc) extracted[1];
}
String udfName = genericUDF.getUdfName();
@@ -187,22 +198,26 @@ private ExprNodeDesc analyzeExpr(
return expr;
}
- ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) extracted[0];
- ExprNodeConstantDesc constantDesc = (ExprNodeConstantDesc) extracted[1];
if (!allowedColumnNames.contains(columnDesc.getColumn())) {
return expr;
}
+ String[] fields = null;
+ if (extracted.length > 2) {
+ fields = ExprNodeDescUtils.extractFields((ExprNodeFieldDesc) extracted[2]);
+ }
+
searchConditions.add(
new IndexSearchCondition(
columnDesc,
udfName,
constantDesc,
- expr));
+ expr,
+ fields));
// we converted the expression to a search condition, so
// remove it from the residual predicate
- return null;
+ return fields == null ? null : expr;
}
/**
@@ -232,4 +247,8 @@ public ExprNodeGenericFuncDesc translateSearchConditions(
}
return expr;
}
+
+ public void setAcceptsFields(boolean acceptsFields) {
+ this.acceptsFields = acceptsFields;
+ }
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java
index 5f1329c..3a2ecb7 100644
--- ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java
+++ ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java
@@ -33,6 +33,16 @@
private ExprNodeConstantDesc constantDesc;
private ExprNodeGenericFuncDesc comparisonExpr;
+ private String[] fields;
+
+ public IndexSearchCondition(
+ ExprNodeColumnDesc columnDesc,
+ String comparisonOp,
+ ExprNodeConstantDesc constantDesc,
+ ExprNodeGenericFuncDesc comparisonExpr) {
+ this(columnDesc, comparisonOp, constantDesc, comparisonExpr, null);
+ }
+
/**
* Constructs a search condition, which takes the form
* column-ref comparison-op constant-value
.
@@ -50,12 +60,14 @@ public IndexSearchCondition(
ExprNodeColumnDesc columnDesc,
String comparisonOp,
ExprNodeConstantDesc constantDesc,
- ExprNodeGenericFuncDesc comparisonExpr) {
+ ExprNodeGenericFuncDesc comparisonExpr,
+ String[] fields) {
this.columnDesc = columnDesc;
this.comparisonOp = comparisonOp;
this.constantDesc = constantDesc;
this.comparisonExpr = comparisonExpr;
+ this.fields = fields;
}
public void setColumnDesc(ExprNodeColumnDesc columnDesc) {
@@ -90,6 +102,10 @@ public ExprNodeGenericFuncDesc getComparisonExpr() {
return comparisonExpr;
}
+ public String[] getFields() {
+ return fields;
+ }
+
@Override
public String toString() {
return comparisonExpr.getExprString();
diff --git ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index 647a9a6..6276680 100755
--- ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -21,6 +21,7 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -413,6 +414,13 @@ public static void pushFilters(JobConf jobConf, TableScanOperator tableScan) {
return;
}
+ Serializable filterObject = scanDesc.getFilterObject();
+ if (filterObject != null) {
+ jobConf.set(
+ TableScanDesc.FILTER_OBJECT_CONF_STR,
+ Utilities.serializeObject(filterObject));
+ }
+
String filterText = filterExpr.getExprString();
String filterExprSerialized = Utilities.serializeExpression(filterExpr);
if (LOG.isDebugEnabled()) {
diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java
index 9f35575..7d7c764 100644
--- ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java
+++ ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java
@@ -23,6 +23,8 @@
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.mapred.JobConf;
+import java.io.Serializable;
+
/**
* HiveStoragePredicateHandler is an optional companion to {@link
* HiveStorageHandler}; it should only be implemented by handlers which
@@ -69,6 +71,11 @@ public DecomposedPredicate decomposePredicate(
public ExprNodeGenericFuncDesc pushedPredicate;
/**
+ * Serialized format for filter
+ */
+ public Serializable pushedPredicateObject;
+
+ /**
* Portion of predicate to be post-evaluated by Hive for any rows
* which are returned by storage handler.
*/
diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
index e50026b..84e8ee8 100644
--- ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
+++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
@@ -260,12 +260,46 @@ private static ExprNodeDesc backtrack(ExprNodeColumnDesc column, Operator> cur
return new ExprNodeDesc[] {expr1, expr2};
}
if (expr1 instanceof ExprNodeConstantDesc && expr2 instanceof ExprNodeColumnDesc) {
- return new ExprNodeDesc[] {expr2, expr1, null}; // add null as a marker (inverted order)
+ return new ExprNodeDesc[] {expr2, expr1};
+ }
+ if (expr1 instanceof ExprNodeFieldDesc && expr2 instanceof ExprNodeConstantDesc) {
+ ExprNodeColumnDesc columnDesc = extractColumn(expr1);
+ return columnDesc != null ? new ExprNodeDesc[] {columnDesc, expr2, expr1} : null;
+ }
+ if (expr1 instanceof ExprNodeConstantDesc && expr2 instanceof ExprNodeFieldDesc) {
+ ExprNodeColumnDesc columnDesc = extractColumn(expr2);
+ return columnDesc != null ? new ExprNodeDesc[] {expr1, columnDesc, expr2} : null;
}
// todo: constant op constant
return null;
}
+ public static String[] extractFields(ExprNodeFieldDesc expr) {
+ return extractFields(expr, new ArrayList()).toArray(new String[0]);
+ }
+
+ private static List extractFields(ExprNodeDesc expr, List fields) {
+ if (expr instanceof ExprNodeFieldDesc) {
+ ExprNodeFieldDesc field = (ExprNodeFieldDesc)expr;
+ fields.add(field.getFieldName());
+ return extractFields(field.getDesc(), fields);
+ }
+ if (expr instanceof ExprNodeColumnDesc) {
+ return fields;
+ }
+ throw new IllegalStateException();
+ }
+
+ private static ExprNodeColumnDesc extractColumn(ExprNodeDesc expr) {
+ if (expr instanceof ExprNodeColumnDesc) {
+ return (ExprNodeColumnDesc)expr;
+ }
+ if (expr instanceof ExprNodeFieldDesc) {
+ return extractColumn(((ExprNodeFieldDesc)expr).getDesc());
+ }
+ return null;
+ }
+
// from IndexPredicateAnalyzer
private static ExprNodeDesc extractConstant(ExprNodeDesc expr) {
if (!(expr instanceof ExprNodeGenericFuncDesc)) {
diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index 10bae4d..8acb39b 100644
--- ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.plan;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -61,6 +62,7 @@
private int maxStatsKeyPrefixLength = -1;
private ExprNodeGenericFuncDesc filterExpr;
+ private transient Serializable filterObject;
public static final String FILTER_EXPR_CONF_STR =
"hive.io.filter.expr.serialized";
@@ -68,6 +70,9 @@
public static final String FILTER_TEXT_CONF_STR =
"hive.io.filter.text";
+ public static final String FILTER_OBJECT_CONF_STR =
+ "hive.io.filter.object";
+
// input file name (big) to bucket number
private Map bucketFileNameMapping;
@@ -110,6 +115,14 @@ public void setFilterExpr(ExprNodeGenericFuncDesc filterExpr) {
this.filterExpr = filterExpr;
}
+ public Serializable getFilterObject() {
+ return filterObject;
+ }
+
+ public void setFilterObject(Serializable filterObject) {
+ this.filterObject = filterObject;
+ }
+
public void setAlias(String alias) {
this.alias = alias;
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
index 40298e1..9813900 100644
--- ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
+++ ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
@@ -836,6 +836,8 @@ private static ExprNodeGenericFuncDesc pushFilterToStorageHandler(
}
}
tableScanDesc.setFilterExpr(decomposed.pushedPredicate);
+ tableScanDesc.setFilterObject(decomposed.pushedPredicateObject);
+
return (ExprNodeGenericFuncDesc)decomposed.residualPredicate;
}
diff --git serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java
new file mode 100644
index 0000000..b7efff0
--- /dev/null
+++ serde/src/java/org/apache/hadoop/hive/serde2/StructObject.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.util.List;
+
+public interface StructObject {
+
+ Object getField(int fieldID);
+
+ List