From cdaa8c0988a758c8c3645f9d15706d2baeb9fbcf Mon Sep 17 00:00:00 2001 From: zhengdong Date: Wed, 23 Nov 2016 16:13:59 +0800 Subject: [PATCH] KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result --- .../java/org/apache/kylin/gridtable/GTUtil.java | 14 +++++++++++++ .../kylin/metadata/filter/CompareTupleFilter.java | 2 ++ .../metadata/filter/TupleFilterSerializer.java | 1 + .../test/resources/query/sql_derived/query12.sql | 24 ++++++++++++++++++++++ .../apache/kylin/query/relnode/OLAPFilterRel.java | 3 +++ .../hbase/common/coprocessor/FilterDecorator.java | 14 +++++++++++++ 6 files changed, 58 insertions(+) mode change 100644 => 100755 core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java mode change 100644 => 100755 core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java mode change 100644 => 100755 core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java create mode 100755 kylin-it/src/test/resources/query/sql_derived/query12.sql mode change 100644 => 100755 query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java mode change 100644 => 100755 storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java old mode 100644 new mode 100755 index 8d310a3..bcd9a2a --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java @@ -198,6 +198,20 @@ public class GTUtil { result = newCompareFilter; } break; + case NOTIN: + Set notInValues = Sets.newHashSet(); + for (Object value : constValues) { + code = translate(col, value, 0); + if (code != null) + notInValues.add(code); + } + if (notInValues.isEmpty()) { + result = ConstantTupleFilter.TRUE; + } else { + newCompareFilter.addChild(new ConstantTupleFilter(notInValues)); + result = newCompareFilter; + } + break; case NEQ: code = translate(col, firstValue, 0); if (code == null) { diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java old mode 100644 new mode 100755 index fe51710..28713ef --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java @@ -59,6 +59,8 @@ public class CompareTupleFilter extends TupleFilter { private CompareTupleFilter(CompareTupleFilter another) { super(new ArrayList(another.children), another.operator); this.column = another.column; + this.firstCondValue = another.getFirstValue(); + this.function = another.getFunction(); this.conditionValues = new HashSet(); this.conditionValues.addAll(another.conditionValues); this.dynamicVariables = new HashMap(); diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java old mode 100644 new mode 100755 index a051ea9..871ca88 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java @@ -166,6 +166,7 @@ public class TupleFilterSerializer { case GT: case GTE: case IN: + case NOTIN: case ISNULL: case ISNOTNULL: filter = new CompareTupleFilter(op); diff --git a/kylin-it/src/test/resources/query/sql_derived/query12.sql b/kylin-it/src/test/resources/query/sql_derived/query12.sql new file mode 100755 index 0000000..01f796f --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_derived/query12.sql @@ -0,0 +1,24 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select UPD_USER,count(1) as CNT +from TEST_KYLIN_FACT as TEST_KYLIN_FACT +JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +where UPD_USER not in ('USER_Y') +group by UPD_USER \ No newline at end of file diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java old mode 100644 new mode 100755 index b34cc6f..002b5af --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java @@ -160,6 +160,9 @@ public class OLAPFilterRel extends Filter implements OLAPRel { if (inFilter != null) { filter = inFilter; } + } else if (op.getKind() == SqlKind.NOT) { + assert (filter.getChildren().size() == 1); + filter = filter.getChildren().get(0).reverse(); } return filter; } diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java old mode 100644 new mode 100755 index ea4b504..cfc199d --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java @@ -91,6 +91,20 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator { result = newCompareFilter; } break; + case NOTIN: + Set notInValues = Sets.newHashSet(); + for (String value : constValues) { + v = translate(col, value, 0); + if (!isDictNull(v)) + notInValues.add(v); + } + if (notInValues.isEmpty()) { + result = ConstantTupleFilter.TRUE; + } else { + newCompareFilter.addChild(new ConstantTupleFilter(notInValues)); + result = newCompareFilter; + } + break; case NEQ: v = translate(col, firstValue, 0); if (isDictNull(v)) { -- 2.8.3