From 714955bb3906d2a3677d8b7d338f2d73f4c778eb Mon Sep 17 00:00:00 2001 From: Zhong Date: Sun, 22 Oct 2017 16:19:41 +0800 Subject: [PATCH 1/2] APACHE-KYLIN-2955: Fix assign a wrong table for find column method in DataModelDesc --- .../apache/kylin/metadata/model/DataModelDesc.java | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index f3def90..4c4d901 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -110,6 +110,8 @@ public class DataModelDesc extends RootPersistentEntity { private Set allTableRefs = Sets.newLinkedHashSet(); private Map aliasMap = Maps.newHashMap(); // alias => TableRef, a table has exactly one alias private Map tableNameMap = Maps.newHashMap(); // name => TableRef, a table maybe referenced by multiple names + private Map> tableDimensionColumns = Maps.newHashMap(); + private Set findColumnExcludeCheckSet = Sets.newHashSet(); private JoinsTree joinsTree; /** @@ -295,6 +297,12 @@ public class DataModelDesc extends RootPersistentEntity { } else { // table not specified, try each table for (TableRef tableRef : allTableRefs) { + if (!findColumnExcludeCheckSet.contains(column)) { + Set columns = tableDimensionColumns.get(tableRef); + if (columns == null || !columns.contains(column)) { + continue; + } + } result = tableRef.getColumn(column); if (result != null) break; @@ -338,6 +346,7 @@ public class DataModelDesc extends RootPersistentEntity { initJoinTablesForUpgrade(); initTableAlias(tables); + initForFindColumn(); initJoinColumns(); reorderJoins(tables); initJoinsTree(); @@ -410,6 +419,31 @@ public class DataModelDesc extends RootPersistentEntity { allTableRefs.addAll(lookupTableRefs); } + private void initForFindColumn() { + for (JoinTableDesc joinTable : joinTables) { + findColumnExcludeCheckSet.addAll(Sets.newHashSet(joinTable.getJoin().getPrimaryKey())); + findColumnExcludeCheckSet.addAll(Sets.newHashSet(joinTable.getJoin().getForeignKey())); + } + findColumnExcludeCheckSet.addAll(Sets.newHashSet(metrics)); + if (!StringUtils.isBlank(partitionDesc.getPartitionDateColumn())) { + findColumnExcludeCheckSet.add(partitionDesc.getPartitionDateColumn()); + } + if (!StringUtils.isBlank(partitionDesc.getPartitionTimeColumn())) { + findColumnExcludeCheckSet.add(partitionDesc.getPartitionTimeColumn()); + } + for (String excludeColumn : Lists.newArrayList(findColumnExcludeCheckSet)) { + findColumnExcludeCheckSet.add(excludeColumn.toUpperCase()); + } + + for (ModelDimensionDesc dim : dimensions) { + Set columns = Sets.newHashSet(dim.getColumns()); + for (String col : dim.getColumns()) { + columns.add(col.toUpperCase()); + } + tableDimensionColumns.put(tableNameMap.get(dim.getTable().toUpperCase()), columns); + } + } + private void addAlias(TableRef ref) { String alias = ref.getAlias(); if (aliasMap.containsKey(alias)) -- 2.5.4 (Apple Git-61) From 1c7a8f902eed87ea5cb430e7d298912714247d91 Mon Sep 17 00:00:00 2001 From: Zhong Date: Fri, 3 Nov 2017 11:34:50 +0800 Subject: [PATCH 2/2] APACHE-KYLIN-2955: add unit test --- .../kylin/metadata/model/DataModelDescTest.java | 8 ++ .../localmeta/model_desc/ut_find_column_model.json | 121 +++++++++++++++++++++ .../test_case_data/localmeta/project/default.json | 3 +- .../localmeta/table/DEFAULT.TEST_FIND_COLUMN.json | 78 +++++++++++++ 4 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 examples/test_case_data/localmeta/model_desc/ut_find_column_model.json create mode 100644 examples/test_case_data/localmeta/table/DEFAULT.TEST_FIND_COLUMN.json diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java index 7feefef..fbd7d55 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java @@ -64,6 +64,14 @@ public class DataModelDescTest extends LocalFileMetadataTestCase { } @Test + public void testFindColumn() { + DataModelDesc model = DataModelManager.getInstance(getTestConfig()).getDataModelDesc("ut_find_column_model"); + assertEquals("TEST_SITES.SITE_NAME", model.findColumn("site_name").getIdentity()); + assertEquals("TEST_FIND_COLUMN.CAL_DT", model.findColumn("cal_dt").getIdentity()); + assertEquals("TEST_FIND_COLUMN.YEAR_BEG_DT", model.findColumn("year_beg_dt").getIdentity()); + } + + @Test public void testGetCopyOf() throws JsonProcessingException { DataModelDesc desc = DataModelManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc"); DataModelDesc copyDesc = DataModelDesc.getCopyOf(desc); diff --git a/examples/test_case_data/localmeta/model_desc/ut_find_column_model.json b/examples/test_case_data/localmeta/model_desc/ut_find_column_model.json new file mode 100644 index 0000000..564cfbe --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/ut_find_column_model.json @@ -0,0 +1,121 @@ +{ + "uuid": "86ed0984-7b22-4a5f-bf02-af4a372ecb41", + "name": "ut_find_column_model", + "lookups": [ + { + "table": "EDW.TEST_CAL_DT", + "join": { + "type": "inner", + "primary_key": [ + "CAL_DT" + ], + "foreign_key": [ + "CAL_DT" + ] + } + }, + { + "table": "DEFAULT.TEST_CATEGORY_GROUPINGS", + "join": { + "type": "inner", + "primary_key": [ + "LEAF_CATEG_ID", + "SITE_ID" + ], + "foreign_key": [ + "LEAF_CATEG_ID", + "LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SITES", + "join": { + "type": "inner", + "primary_key": [ + "SITE_ID" + ], + "foreign_key": [ + "LSTG_SITE_ID" + ] + } + }, + { + "table": "EDW.TEST_SELLER_TYPE_DIM", + "join": { + "type": "inner", + "primary_key": [ + "SELLER_TYPE_CD" + ], + "foreign_key": [ + "SLR_SEGMENT_CD" + ] + } + } + ], + "dimensions": [ + { + "table": "default.test_find_column", + "columns": [ + "lstg_format_name", + "LSTG_SITE_ID", + "SLR_SEGMENT_CD", + "TRANS_ID", + "CAL_DT", + "LEAF_CATEG_ID", + "SELLER_ID", + "ORDER_ID" + ] + }, + { + "table": "default.test_category_groupings", + "columns": [ + "leaf_categ_id", + "site_id", + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "UPD_DATE", + "UPD_USER", + "meta_categ_name", + "categ_lvl2_name", + "categ_lvl3_name" + ] + }, + { + "table": "edw.test_sites", + "columns": [ + "site_id", + "site_name", + "cre_user" + ] + }, + { + "table": "edw.test_seller_type_dim", + "columns": [ + "seller_type_cd", + "seller_type_desc" + ] + }, + { + "table": "edw.test_cal_dt", + "columns": [ + "cal_dt", + "week_beg_dt" + ] + } + ], + "metrics": [ + "edw.test_cal_dt.cal_dt", + "PRICE", + "ITEM_COUNT", + "YEAR_BEG_DT" + ], + "last_modified": 1422435345352, + "fact_table": "DEFAULT.TEST_FIND_COLUMN", + "filter_condition": null, + "partition_desc": { + "partition_date_column": "DEFAULT.TEST_FIND_COLUMN.cal_dt", + "partition_date_start": 0, + "partition_type": "APPEND" + } +} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json index 7b5fd23..ae29dcf 100644 --- a/examples/test_case_data/localmeta/project/default.json +++ b/examples/test_case_data/localmeta/project/default.json @@ -78,7 +78,8 @@ "test_streaming_table_model_desc", "fifty_dim", "twenty_dim", - "ut_large_dimension_number" + "ut_large_dimension_number", + "ut_find_column_model" ], "override_kylin_properties" :{ "kylin.storage.hbase.owner-tag": "kylin@kylin.apache.org" diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_FIND_COLUMN.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_FIND_COLUMN.json new file mode 100644 index 0000000..85a0bd2 --- /dev/null +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_FIND_COLUMN.json @@ -0,0 +1,78 @@ +{ + "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771", + "name" : "TEST_FIND_COLUMN", + "data_gen" : "1", + "columns" : [ { + "id" : "1", + "name" : "TRANS_ID", + "datatype" : "bigint", + "data_gen" : "ID" + }, { + "id" : "2", + "name" : "ORDER_ID", + "datatype" : "bigint", + "index": "T" + }, { + "id" : "3", + "name" : "CAL_DT", + "datatype" : "date", + "data_gen" : "FK,order", + "index": "T" + }, { + "id" : "4", + "name" : "YEAR_BEG_DT", + "datatype" : "date", + "index": "T" + }, { + "id" : "5", + "name" : "LSTG_FORMAT_NAME", + "datatype" : "string", + "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others", + "index": "T" + }, { + "id" : "6", + "name" : "LEAF_CATEG_ID", + "datatype" : "bigint", + "data_gen" : "FK,null,nullstr=0", + "index": "T" + }, { + "id" : "7", + "name" : "LSTG_SITE_ID", + "datatype" : "int", + "index": "T" + }, { + "id" : "8", + "name" : "SITE_NAME", + "datatype" : "string", + "index": "T" + }, { + "id" : "9", + "name" : "SLR_SEGMENT_CD", + "datatype" : "smallint", + "data_gen" : "FK,pk=EDW.TEST_SELLER_TYPE_DIM_TABLE.SELLER_TYPE_CD", + "index": "T" + }, { + "id" : "10", + "name" : "SELLER_ID", + "datatype" : "int", + "data_gen" : "RAND||10000000|10001000", + "index": "T" + }, { + "id" : "11", + "name" : "PRICE", + "datatype" : "decimal", + "data_gen" : "RAND|.##|-100|1000" + }, { + "id" : "12", + "name" : "ITEM_COUNT", + "datatype" : "int", + "data_gen" : "RAND" + }, { + "id" : "13", + "name" : "TEST_COUNT_DISTINCT_BITMAP", + "datatype" : "string", + "data_gen" : "RAND" + } ], + "database" : "DEFAULT", + "last_modified" : 0 +} -- 2.5.4 (Apple Git-61)