From 847f3859476baac5939fb3540e6d17414e109757 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Wed, 3 Dec 2014 10:58:36 +0800 Subject: [PATCH 1/3] fix NPE --- .../java/com/kylinolap/job/cmd/JavaHadoopCmdOutput.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/job/src/main/java/com/kylinolap/job/cmd/JavaHadoopCmdOutput.java b/job/src/main/java/com/kylinolap/job/cmd/JavaHadoopCmdOutput.java index 20dd9dd..c3f9236 100644 --- a/job/src/main/java/com/kylinolap/job/cmd/JavaHadoopCmdOutput.java +++ b/job/src/main/java/com/kylinolap/job/cmd/JavaHadoopCmdOutput.java @@ -18,6 +18,7 @@ import java.util.Map; +import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.TaskCounter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -159,16 +160,23 @@ private void updateHadoopJobInfo() { private void updateJobCounter() { try { - this.output.append(job.getCounters().toString()).append("\n"); - log.debug(job.getCounters().toString()); + Counters counters = job.getCounters(); + if (counters == null) { + String errorMsg = "no counters for job " + mrJobID; + log.warn(errorMsg); + output.append(errorMsg); + return; + } + this.output.append(counters.toString()).append("\n"); + log.debug(counters.toString()); JobDAO jobDAO = JobDAO.getInstance(config); JobInstance jobInstance = jobDAO.getJob(jobInstanceID); JobStep jobStep = jobInstance.getSteps().get(jobStepID); - long mapInputRecords = job.getCounters().findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue(); + long mapInputRecords = counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue(); jobStep.putInfo(JobInstance.SOURCE_RECORDS_COUNT, String.valueOf(mapInputRecords)); - long hdfsBytesWritten = job.getCounters().findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); + long hdfsBytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); jobStep.putInfo(JobInstance.HDFS_BYTES_WRITTEN, String.valueOf(hdfsBytesWritten)); jobDAO.updateJobInstance(jobInstance); From 7877dfddc84b51e7d9b0a7b4782662161d496b2b Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Wed, 3 Dec 2014 16:58:11 +0800 Subject: [PATCH 2/3] use [db.table] to distinguish tables with same name in different db --- .../kylinolap/cube/project/ProjectInstance.java | 7 ++-- .../com/kylinolap/cube/project/ProjectManager.java | 26 +++++++------- .../com/kylinolap/metadata/MetadataManager.java | 31 +++++++++------- .../com/kylinolap/metadata/model/TableDesc.java | 21 +++++++++-- .../kylinolap/metadata/model/TableDescTest.java | 41 ++++++++++++++++++++++ 5 files changed, 94 insertions(+), 32 deletions(-) create mode 100644 metadata/src/test/java/com/kylinolap/metadata/model/TableDescTest.java diff --git a/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java b/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java index f8939e4..26d4c90 100644 --- a/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java +++ b/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java @@ -182,14 +182,15 @@ public int getTablesCount() { } public void addTable(String tableName) { - tableName = tableName.toUpperCase(); + tableName = tableName.toUpperCase(); this.getTables().add(tableName); } //will return new Set for null public Set getTables() { - tables = tables==null?new TreeSet():tables; - return tables; } + tables = tables == null ? new TreeSet() : tables; + return tables; + } public String getOwner() { return owner; diff --git a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java b/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java index fba564a..2a9004e 100644 --- a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java +++ b/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java @@ -17,11 +17,7 @@ package com.kylinolap.cube.project; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.lang3.StringUtils; @@ -243,7 +239,7 @@ public void removeCubeFromProjects(String cubeName) throws IOException { //sync exposed table to project when list List exposedTables = listExposedTables(project); for (TableDesc table : exposedTables) { - projectInstance.addTable(table.getName()); + projectInstance.addTable(TableDesc.getTableIdentity(table)); } //only save project json if new tables are sync in if (originTableCount < projectInstance.getTablesCount()) { @@ -420,8 +416,9 @@ private synchronized ProjectInstance loadProject(String path, boolean triggerUpd ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER); projectInstance.init(); - if (StringUtils.isBlank(projectInstance.getName())) + if (StringUtils.isBlank(projectInstance.getName())) { throw new IllegalStateException("Project name must not be blank"); + } if (triggerUpdate) { projectMap.put(projectInstance.getName().toUpperCase(), projectInstance); @@ -532,15 +529,17 @@ private void markExposedTablesAndColumns(String projectName, CubeInstance cubeIn private void markExposedTableAndColumn(String project, String table, String column, Object refObj) { project = ProjectInstance.getNormalizedProjectName(project); TableDesc t = this.getMetadataManager().getTableDesc(table); - if (t == null) + if (t == null) { throw new IllegalStateException("No SourceTable found by name '" + table + "', ref by " + refObj); + } table = t.getName(); // ensures upper case ProjectTable projTable = getProjectTable(project, table, true); ColumnDesc srcCol = t.findColumnByName(column); - if (srcCol == null) + if (srcCol == null) { throw new IllegalStateException("No SourceColumn found by name '" + table + "/" + column + "', ref by " + refObj); + } if (!projTable.getColumns().contains(srcCol.getName())) { projTable.getColumns().add(srcCol.getName()); @@ -556,11 +555,10 @@ private ProjectTable getProjectTable(String project, final String table, boolean project = ProjectInstance.getNormalizedProjectName(project); if (this.projectTables.containsEntry(project, new ProjectTable(table))) { - Iterator projsIter = this.projectTables.get(project).iterator(); - while (projsIter.hasNext()) { - ProjectTable oneTable = projsIter.next(); - if (oneTable.getName().equalsIgnoreCase(table)) { - projectTable = oneTable; + Collection projects = this.projectTables.get(project); + for (ProjectTable pt : projects) { + if (pt.getName().equalsIgnoreCase(table)) { + projectTable = pt; break; } } diff --git a/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java b/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java index 31a0b2e..53702ac 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java +++ b/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java @@ -140,8 +140,7 @@ public ResourceStore getStore() { * @return */ public TableDesc getTableDesc(String tableName) { - tableName = tableName.toUpperCase(); - return srcTableMap.get(tableName); + return srcTableMap.get(TableDesc.getTableIdentity(tableName)); } /** @@ -151,10 +150,10 @@ public TableDesc getTableDesc(String tableName) { * @return */ public Map getTableDescExd(String tableName) { - tableName = tableName.toUpperCase(); + String tableIdentity = TableDesc.getTableIdentity(tableName); Map result = new HashMap(); - if (srcTableExdMap.containsKey(tableName)) { - Map tmp = srcTableExdMap.get(tableName); + if (srcTableExdMap.containsKey(tableIdentity)) { + Map tmp = srcTableExdMap.get(tableIdentity); Iterator> it = tmp.entrySet().iterator(); while (it.hasNext()) { Entry entry = it.next(); @@ -168,15 +167,18 @@ public TableDesc getTableDesc(String tableName) { } public void createSourceTable(TableDesc srcTable) throws IOException { - if (srcTable.getUuid() == null || srcTable.getName() == null) + if (srcTable.getUuid() == null || srcTable.getName() == null) { throw new IllegalArgumentException(); - if (srcTableMap.containsKey(srcTable.getName())) + } + String tableIdentity = TableDesc.getTableIdentity(srcTable); + if (srcTableMap.containsKey(tableIdentity)) { throw new IllegalArgumentException("SourceTable '" + srcTable.getName() + "' already exists"); + } String path = srcTable.getResourcePath(); getStore().putResource(path, srcTable, TABLE_SERIALIZER); - srcTableMap.put(srcTable.getName(), srcTable); + srcTableMap.put(tableIdentity, srcTable); } public InvertedIndexDesc getInvertedIndexDesc(String name) { @@ -202,7 +204,7 @@ private void reloadAllSourceTableExd() throws IOException { for (String path : paths) { Map attrContainer = new HashMap(); String tableName = loadSourceTableExd(getStore(), path, attrContainer); - srcTableExdMap.putLocal(tableName.toUpperCase(), attrContainer); + srcTableExdMap.putLocal(TableDesc.getTableIdentity(tableName), attrContainer); } logger.debug("Loaded " + paths.size() + " SourceTable EXD(s)"); } @@ -249,12 +251,15 @@ private TableDesc loadSourceTable(String path) throws IOException { TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER); t.init(); - if (StringUtils.isBlank(t.getName())) + String tableIdentity = TableDesc.getTableIdentity(t); + if (StringUtils.isBlank(tableIdentity)) { throw new IllegalStateException("SourceTable name must not be blank"); - if (srcTableMap.containsKey(t.getName())) - throw new IllegalStateException("Dup SourceTable name '" + t.getName() + "'"); + } + if (srcTableMap.containsKey(tableIdentity)) { + throw new IllegalStateException("Dup SourceTable name '" + tableIdentity + "'"); + } - srcTableMap.putLocal(t.getName(), t); + srcTableMap.putLocal(tableIdentity, t); return t; } diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/TableDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/TableDesc.java index 917cc60..f1145bf 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/model/TableDesc.java +++ b/metadata/src/main/java/com/kylinolap/metadata/model/TableDesc.java @@ -17,6 +17,7 @@ import java.util.Arrays; import java.util.Comparator; +import java.util.regex.Pattern; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; @@ -56,7 +57,7 @@ public ColumnDesc findColumnByName(String name) { } public String getResourcePath() { - return ResourceStore.TABLE_RESOURCE_ROOT + "/" + name + ".json"; + return ResourceStore.TABLE_RESOURCE_ROOT + "/" + getTableIdentity(this) + ".json"; } // ============================================================================ @@ -129,7 +130,23 @@ public int compare(ColumnDesc col1, ColumnDesc col2) { @Override public String toString() { - return "TableDesc [name=" + name + "]"; + return "TableDesc [database=" + getDatabase() + " name=" + name + "]"; + } + + private static final Pattern TABLE_IDENTITY_PATTERN = Pattern.compile("^\\w+\\.\\w+$"); + + public static String getTableIdentity(TableDesc tableDesc) { + return String.format("%s.%s", tableDesc.getDatabase(), tableDesc.getName()).toUpperCase(); + } + + public static String getTableIdentity(String tableName) { + if (!tableName.contains(".")) { + tableName = "DEFAULT." + tableName; + } + if (!TABLE_IDENTITY_PATTERN.matcher(tableName).matches()) { + throw new IllegalArgumentException("invalid tableName:" + tableName); + } + return tableName.toUpperCase(); } } diff --git a/metadata/src/test/java/com/kylinolap/metadata/model/TableDescTest.java b/metadata/src/test/java/com/kylinolap/metadata/model/TableDescTest.java new file mode 100644 index 0000000..d8a0955 --- /dev/null +++ b/metadata/src/test/java/com/kylinolap/metadata/model/TableDescTest.java @@ -0,0 +1,41 @@ +package com.kylinolap.metadata.model; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Created by qianzhou on 12/3/14. + */ +public class TableDescTest { + + @Test + public void testTableIdentity() { + final String tableName = "testtable"; + final String dbName = "testdb"; + TableDesc table = new TableDesc(); + table.setName(tableName); + assertEquals(("DEFAULT." + tableName).toUpperCase(), TableDesc.getTableIdentity(table)); + assertEquals(("DEFAULT." + tableName).toUpperCase(), TableDesc.getTableIdentity(tableName)); + assertEquals(("DEFAULT." + tableName).toUpperCase(), TableDesc.getTableIdentity("DEFAULT." + tableName)); + assertEquals((dbName + "." + tableName).toUpperCase(), TableDesc.getTableIdentity(dbName + "." + tableName)); + + table.setDatabase(dbName); + assertEquals((dbName + "." + tableName).toUpperCase(), TableDesc.getTableIdentity(dbName + "." + tableName)); + + try { + TableDesc.getTableIdentity("1 2"); + fail("should throw IllegalArgumentException"); + } catch (IllegalArgumentException ex) { + System.out.println(ex.getMessage()); + } + try { + TableDesc.getTableIdentity("t.2.abc"); + fail("should throw IllegalArgumentException"); + } catch (IllegalArgumentException ex) { + System.out.println(ex.getMessage()); + } + + } +} From 441add8414ba5990d321a8f409c0c9f653d1f317 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Wed, 3 Dec 2014 17:41:37 +0800 Subject: [PATCH 3/3] refactor example --- .../model_desc/test_kylin_ii_model_desc.json | 10 +- .../test_kylin_with_slr_left_join_model_desc.json | 12 +- .../model_desc/test_kylin_with_slr_model_desc.json | 10 +- ...est_kylin_without_slr_left_join_model_desc.json | 10 +- .../test_kylin_without_slr_model_desc.json | 10 +- .../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json | 151 ++++++++ .../localmeta/table/DEFAULT.TEST_KYLIN_FACT.json | 43 +++ .../localmeta/table/EDW.TEST_CAL_DT.json | 407 +++++++++++++++++++++ .../localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json | 43 +++ .../localmeta/table/EDW.TEST_SITES.json | 47 +++ .../localmeta/table/TEST_CAL_DT.json | 407 --------------------- .../localmeta/table/TEST_CATEGORY_GROUPINGS.json | 151 -------- .../localmeta/table/TEST_KYLIN_FACT.json | 43 --- .../localmeta/table/TEST_SELLER_TYPE_DIM.json | 43 --- .../test_case_data/localmeta/table/TEST_SITES.json | 47 --- .../kylinolap/metadata/model/DataModelDesc.java | 5 +- 16 files changed, 719 insertions(+), 720 deletions(-) create mode 100644 examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json create mode 100644 examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json create mode 100644 examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json create mode 100644 examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json create mode 100644 examples/test_case_data/localmeta/table/EDW.TEST_SITES.json delete mode 100644 examples/test_case_data/localmeta/table/TEST_CAL_DT.json delete mode 100644 examples/test_case_data/localmeta/table/TEST_CATEGORY_GROUPINGS.json delete mode 100644 examples/test_case_data/localmeta/table/TEST_KYLIN_FACT.json delete mode 100644 examples/test_case_data/localmeta/table/TEST_SELLER_TYPE_DIM.json delete mode 100644 examples/test_case_data/localmeta/table/TEST_SITES.json diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json index 6c2d7b1..6859e04 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json @@ -1,9 +1,9 @@ { "name" : "test_kylin_ii_model_desc", - "fact_table": "TEST_KYLIN_FACT", + "fact_table": "DEFAULT.TEST_KYLIN_FACT", "lookups": [ { - "table" : "TEST_CAL_DT", + "table" : "EDW.TEST_CAL_DT", "join" : { "type" : "left", "primary_key" : [ "CAL_DT" ], @@ -11,7 +11,7 @@ } }, { - "table" : "TEST_CATEGORY_GROUPINGS", + "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS", "join" : { "type" : "left", "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], @@ -19,7 +19,7 @@ } }, { - "table" : "TEST_SITES", + "table" : "EDW.TEST_SITES", "join" : { "type" : "left", "primary_key" : [ "SITE_ID" ], @@ -27,7 +27,7 @@ } }, { - "table" : "TEST_SELLER_TYPE_DIM", + "table" : "EDW.TEST_SELLER_TYPE_DIM", "join" : { "type" : "left", "primary_key" : [ "SELLER_TYPE_CD" ], diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json index acb8e0c..ca72e05 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json @@ -1,9 +1,9 @@ { "name": "test_kylin_with_slr_left_join_model_desc", - "fact_table": "test_kylin_fact", + "fact_table": "DEFAULT.test_kylin_fact", "lookups": [ { - "table": "test_cal_dt", + "table": "EDW.test_cal_dt", "join": { "type": "left", "primary_key": ["cal_dt"], @@ -11,7 +11,7 @@ } }, { - "table": "test_category_groupings", + "table": "DEFAULT.test_category_groupings", "join": { "type": "left", "primary_key": ["leaf_categ_id", "site_id"], @@ -19,7 +19,7 @@ } }, { - "table": "test_category_groupings", + "table": "DEFAULT.test_category_groupings", "join": { "type": "left", "primary_key": ["leaf_categ_id", "site_id"], @@ -27,7 +27,7 @@ } }, { - "table": "test_sites", + "table": "EDW.test_sites", "join": { "type": "left", "primary_key": ["site_id"], @@ -35,7 +35,7 @@ } }, { - "table": "test_seller_type_dim", + "table": "EDW.test_seller_type_dim", "join": { "type": "left", "primary_key": ["seller_type_cd"], diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json index b1347a1..63404df 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json @@ -1,9 +1,9 @@ { "name": "test_kylin_with_slr_model_desc", - "fact_table": "test_kylin_fact", + "fact_table": "DEFAULT.test_kylin_fact", "lookups": [ { - "table": "test_cal_dt", + "table": "EDW.test_cal_dt", "join": { "type": "inner", "primary_key": ["cal_dt"], @@ -11,7 +11,7 @@ } }, { - "table": "test_category_groupings", + "table": "DEFAULT.test_category_groupings", "join": { "type": "inner", "primary_key": ["leaf_categ_id", "site_id"], @@ -19,7 +19,7 @@ } }, { - "table": "test_sites", + "table": "EDW.test_sites", "join": { "type": "inner", "primary_key": ["site_id"], @@ -27,7 +27,7 @@ } }, { - "table": "test_seller_type_dim", + "table": "EDW.test_seller_type_dim", "join": { "type": "inner", "primary_key": ["seller_type_cd"], diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json index 5773190..a3380d0 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json @@ -1,9 +1,9 @@ { "name": "test_kylin_without_slr_left_join_model_desc", - "fact_table": "test_kylin_fact", + "fact_table": "DEFAULT.test_kylin_fact", "lookups": [ { - "table": "test_cal_dt", + "table": "EDW.test_cal_dt", "join": { "type": "left", "primary_key": [ @@ -15,7 +15,7 @@ } }, { - "table": "test_category_groupings", + "table": "DEFAULT.test_category_groupings", "join": { "type": "left", "primary_key": [ @@ -29,7 +29,7 @@ } }, { - "table": "test_sites", + "table": "EDW.test_sites", "join": { "type": "left", "primary_key": [ @@ -41,7 +41,7 @@ } }, { - "table": "test_seller_type_dim", + "table": "EDW.test_seller_type_dim", "join": { "type": "left", "primary_key": [ diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json index 09319ab..8775b23 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json @@ -1,9 +1,9 @@ { "name": "test_kylin_without_slr_model_desc", - "fact_table": "test_kylin_fact", + "fact_table": "DEFAULT.test_kylin_fact", "lookups": [ { - "table" : "test_cal_dt", + "table" : "EDW.test_cal_dt", "join": { "type": "inner", "primary_key": [ @@ -15,7 +15,7 @@ } }, { - "table": "test_category_groupings", + "table": "DEFAULT.test_category_groupings", "join": { "type": "inner", "primary_key": [ @@ -29,7 +29,7 @@ } }, { - "table": "test_sites", + "table": "EDW.test_sites", "join": { "type": "inner", "primary_key": [ @@ -41,7 +41,7 @@ } }, { - "table": "test_seller_type_dim", + "table": "EDW.test_seller_type_dim", "join": { "type": "inner", "primary_key": [ diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json new file mode 100644 index 0000000..7bcd092 --- /dev/null +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json @@ -0,0 +1,151 @@ +{ + "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f", + "name" : "TEST_CATEGORY_GROUPINGS", + "columns" : [ { + "id" : "1", + "name" : "LEAF_CATEG_ID", + "datatype" : "bigint" + }, { + "id" : "2", + "name" : "LEAF_CATEG_NAME", + "datatype" : "string" + }, { + "id" : "3", + "name" : "SITE_ID", + "datatype" : "int" + }, { + "id" : "4", + "name" : "CATEG_BUSN_MGR", + "datatype" : "string" + }, { + "id" : "5", + "name" : "CATEG_BUSN_UNIT", + "datatype" : "string" + }, { + "id" : "6", + "name" : "REGN_CATEG", + "datatype" : "string" + }, { + "id" : "7", + "name" : "USER_DEFINED_FIELD1", + "datatype" : "string" + }, { + "id" : "8", + "name" : "USER_DEFINED_FIELD3", + "datatype" : "string" + }, { + "id" : "9", + "name" : "CRE_DATE", + "datatype" : "string" + }, { + "id" : "10", + "name" : "UPD_DATE", + "datatype" : "string" + }, { + "id" : "11", + "name" : "CRE_USER", + "datatype" : "string" + }, { + "id" : "12", + "name" : "UPD_USER", + "datatype" : "string" + }, { + "id" : "13", + "name" : "META_CATEG_ID", + "datatype" : "decimal" + }, { + "id" : "14", + "name" : "META_CATEG_NAME", + "datatype" : "string" + }, { + "id" : "15", + "name" : "CATEG_LVL2_ID", + "datatype" : "decimal" + }, { + "id" : "16", + "name" : "CATEG_LVL3_ID", + "datatype" : "decimal" + }, { + "id" : "17", + "name" : "CATEG_LVL4_ID", + "datatype" : "decimal" + }, { + "id" : "18", + "name" : "CATEG_LVL5_ID", + "datatype" : "decimal" + }, { + "id" : "19", + "name" : "CATEG_LVL6_ID", + "datatype" : "decimal" + }, { + "id" : "20", + "name" : "CATEG_LVL7_ID", + "datatype" : "decimal" + }, { + "id" : "21", + "name" : "CATEG_LVL2_NAME", + "datatype" : "string" + }, { + "id" : "22", + "name" : "CATEG_LVL3_NAME", + "datatype" : "string" + }, { + "id" : "23", + "name" : "CATEG_LVL4_NAME", + "datatype" : "string" + }, { + "id" : "24", + "name" : "CATEG_LVL5_NAME", + "datatype" : "string" + }, { + "id" : "25", + "name" : "CATEG_LVL6_NAME", + "datatype" : "string" + }, { + "id" : "26", + "name" : "CATEG_LVL7_NAME", + "datatype" : "string" + }, { + "id" : "27", + "name" : "CATEG_FLAGS", + "datatype" : "decimal" + }, { + "id" : "28", + "name" : "ADULT_CATEG_YN", + "datatype" : "string" + }, { + "id" : "29", + "name" : "DOMAIN_ID", + "datatype" : "decimal" + }, { + "id" : "30", + "name" : "USER_DEFINED_FIELD5", + "datatype" : "string" + }, { + "id" : "31", + "name" : "VCS_ID", + "datatype" : "decimal" + }, { + "id" : "32", + "name" : "GCS_ID", + "datatype" : "decimal" + }, { + "id" : "33", + "name" : "MOVE_TO", + "datatype" : "decimal" + }, { + "id" : "34", + "name" : "SAP_CATEGORY_ID", + "datatype" : "decimal" + }, { + "id" : "35", + "name" : "SRC_ID", + "datatype" : "tinyint" + }, { + "id" : "36", + "name" : "BSNS_VRTCL_NAME", + "datatype" : "string" + } ], + "database" : "DEFAULT", + "last_modified" : 0 +} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json new file mode 100644 index 0000000..732351a --- /dev/null +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json @@ -0,0 +1,43 @@ +{ + "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771", + "name" : "TEST_KYLIN_FACT", + "columns" : [ { + "id" : "1", + "name" : "TRANS_ID", + "datatype" : "bigint" + }, { + "id" : "2", + "name" : "CAL_DT", + "datatype" : "date" + }, { + "id" : "3", + "name" : "LSTG_FORMAT_NAME", + "datatype" : "string" + }, { + "id" : "4", + "name" : "LEAF_CATEG_ID", + "datatype" : "bigint" + }, { + "id" : "5", + "name" : "LSTG_SITE_ID", + "datatype" : "int" + }, { + "id" : "6", + "name" : "SLR_SEGMENT_CD", + "datatype" : "smallint" + }, { + "id" : "7", + "name" : "PRICE", + "datatype" : "decimal(18,6)" + }, { + "id" : "8", + "name" : "ITEM_COUNT", + "datatype" : "bigint" + }, { + "id" : "9", + "name" : "SELLER_ID", + "datatype" : "bigint" + } ], + "database" : "DEFAULT", + "last_modified" : 0 +} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json new file mode 100644 index 0000000..0ad7ee1 --- /dev/null +++ b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json @@ -0,0 +1,407 @@ +{ + "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a", + "name" : "TEST_CAL_DT", + "columns" : [ { + "id" : "1", + "name" : "CAL_DT", + "datatype" : "date" + }, { + "id" : "2", + "name" : "YEAR_BEG_DT", + "datatype" : "date" + }, { + "id" : "3", + "name" : "QTR_BEG_DT", + "datatype" : "date" + }, { + "id" : "4", + "name" : "MONTH_BEG_DT", + "datatype" : "date" + }, { + "id" : "5", + "name" : "WEEK_BEG_DT", + "datatype" : "date" + }, { + "id" : "6", + "name" : "AGE_FOR_YEAR_ID", + "datatype" : "smallint" + }, { + "id" : "7", + "name" : "AGE_FOR_QTR_ID", + "datatype" : "smallint" + }, { + "id" : "8", + "name" : "AGE_FOR_MONTH_ID", + "datatype" : "smallint" + }, { + "id" : "9", + "name" : "AGE_FOR_WEEK_ID", + "datatype" : "smallint" + }, { + "id" : "10", + "name" : "AGE_FOR_DT_ID", + "datatype" : "smallint" + }, { + "id" : "11", + "name" : "AGE_FOR_RTL_YEAR_ID", + "datatype" : "smallint" + }, { + "id" : "12", + "name" : "AGE_FOR_RTL_QTR_ID", + "datatype" : "smallint" + }, { + "id" : "13", + "name" : "AGE_FOR_RTL_MONTH_ID", + "datatype" : "smallint" + }, { + "id" : "14", + "name" : "AGE_FOR_RTL_WEEK_ID", + "datatype" : "smallint" + }, { + "id" : "15", + "name" : "AGE_FOR_CS_WEEK_ID", + "datatype" : "smallint" + }, { + "id" : "16", + "name" : "DAY_OF_CAL_ID", + "datatype" : "int" + }, { + "id" : "17", + "name" : "DAY_OF_YEAR_ID", + "datatype" : "smallint" + }, { + "id" : "18", + "name" : "DAY_OF_QTR_ID", + "datatype" : "smallint" + }, { + "id" : "19", + "name" : "DAY_OF_MONTH_ID", + "datatype" : "smallint" + }, { + "id" : "20", + "name" : "DAY_OF_WEEK_ID", + "datatype" : "int" + }, { + "id" : "21", + "name" : "WEEK_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "22", + "name" : "WEEK_OF_CAL_ID", + "datatype" : "int" + }, { + "id" : "23", + "name" : "MONTH_OF_QTR_ID", + "datatype" : "tinyint" + }, { + "id" : "24", + "name" : "MONTH_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "25", + "name" : "MONTH_OF_CAL_ID", + "datatype" : "smallint" + }, { + "id" : "26", + "name" : "QTR_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "27", + "name" : "QTR_OF_CAL_ID", + "datatype" : "smallint" + }, { + "id" : "28", + "name" : "YEAR_OF_CAL_ID", + "datatype" : "smallint" + }, { + "id" : "29", + "name" : "YEAR_END_DT", + "datatype" : "string" + }, { + "id" : "30", + "name" : "QTR_END_DT", + "datatype" : "string" + }, { + "id" : "31", + "name" : "MONTH_END_DT", + "datatype" : "string" + }, { + "id" : "32", + "name" : "WEEK_END_DT", + "datatype" : "string" + }, { + "id" : "33", + "name" : "CAL_DT_NAME", + "datatype" : "string" + }, { + "id" : "34", + "name" : "CAL_DT_DESC", + "datatype" : "string" + }, { + "id" : "35", + "name" : "CAL_DT_SHORT_NAME", + "datatype" : "string" + }, { + "id" : "36", + "name" : "YTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "37", + "name" : "QTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "38", + "name" : "MTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "39", + "name" : "WTD_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "40", + "name" : "SEASON_BEG_DT", + "datatype" : "string" + }, { + "id" : "41", + "name" : "DAY_IN_YEAR_COUNT", + "datatype" : "smallint" + }, { + "id" : "42", + "name" : "DAY_IN_QTR_COUNT", + "datatype" : "tinyint" + }, { + "id" : "43", + "name" : "DAY_IN_MONTH_COUNT", + "datatype" : "tinyint" + }, { + "id" : "44", + "name" : "DAY_IN_WEEK_COUNT", + "datatype" : "tinyint" + }, { + "id" : "45", + "name" : "RTL_YEAR_BEG_DT", + "datatype" : "string" + }, { + "id" : "46", + "name" : "RTL_QTR_BEG_DT", + "datatype" : "string" + }, { + "id" : "47", + "name" : "RTL_MONTH_BEG_DT", + "datatype" : "string" + }, { + "id" : "48", + "name" : "RTL_WEEK_BEG_DT", + "datatype" : "string" + }, { + "id" : "49", + "name" : "CS_WEEK_BEG_DT", + "datatype" : "string" + }, { + "id" : "50", + "name" : "CAL_DATE", + "datatype" : "string" + }, { + "id" : "51", + "name" : "DAY_OF_WEEK", + "datatype" : "string" + }, { + "id" : "52", + "name" : "MONTH_ID", + "datatype" : "string" + }, { + "id" : "53", + "name" : "PRD_DESC", + "datatype" : "string" + }, { + "id" : "54", + "name" : "PRD_FLAG", + "datatype" : "string" + }, { + "id" : "55", + "name" : "PRD_ID", + "datatype" : "string" + }, { + "id" : "56", + "name" : "PRD_IND", + "datatype" : "string" + }, { + "id" : "57", + "name" : "QTR_DESC", + "datatype" : "string" + }, { + "id" : "58", + "name" : "QTR_ID", + "datatype" : "string" + }, { + "id" : "59", + "name" : "QTR_IND", + "datatype" : "string" + }, { + "id" : "60", + "name" : "RETAIL_WEEK", + "datatype" : "string" + }, { + "id" : "61", + "name" : "RETAIL_YEAR", + "datatype" : "string" + }, { + "id" : "62", + "name" : "RETAIL_START_DATE", + "datatype" : "string" + }, { + "id" : "63", + "name" : "RETAIL_WK_END_DATE", + "datatype" : "string" + }, { + "id" : "64", + "name" : "WEEK_IND", + "datatype" : "string" + }, { + "id" : "65", + "name" : "WEEK_NUM_DESC", + "datatype" : "string" + }, { + "id" : "66", + "name" : "WEEK_BEG_DATE", + "datatype" : "string" + }, { + "id" : "67", + "name" : "WEEK_END_DATE", + "datatype" : "string" + }, { + "id" : "68", + "name" : "WEEK_IN_YEAR_ID", + "datatype" : "string" + }, { + "id" : "69", + "name" : "WEEK_ID", + "datatype" : "string" + }, { + "id" : "70", + "name" : "WEEK_BEG_END_DESC_MDY", + "datatype" : "string" + }, { + "id" : "71", + "name" : "WEEK_BEG_END_DESC_MD", + "datatype" : "string" + }, { + "id" : "72", + "name" : "YEAR_ID", + "datatype" : "string" + }, { + "id" : "73", + "name" : "YEAR_IND", + "datatype" : "string" + }, { + "id" : "74", + "name" : "CAL_DT_MNS_1YEAR_DT", + "datatype" : "string" + }, { + "id" : "75", + "name" : "CAL_DT_MNS_2YEAR_DT", + "datatype" : "string" + }, { + "id" : "76", + "name" : "CAL_DT_MNS_1QTR_DT", + "datatype" : "string" + }, { + "id" : "77", + "name" : "CAL_DT_MNS_2QTR_DT", + "datatype" : "string" + }, { + "id" : "78", + "name" : "CAL_DT_MNS_1MONTH_DT", + "datatype" : "string" + }, { + "id" : "79", + "name" : "CAL_DT_MNS_2MONTH_DT", + "datatype" : "string" + }, { + "id" : "80", + "name" : "CAL_DT_MNS_1WEEK_DT", + "datatype" : "string" + }, { + "id" : "81", + "name" : "CAL_DT_MNS_2WEEK_DT", + "datatype" : "string" + }, { + "id" : "82", + "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "83", + "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "84", + "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "85", + "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "86", + "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "87", + "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "88", + "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND", + "datatype" : "tinyint" + }, { + "id" : "89", + "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND", + "datatype" : "tinyint" + }, { + "id" : "90", + "name" : "RTL_MONTH_OF_RTL_YEAR_ID", + "datatype" : "string" + }, { + "id" : "91", + "name" : "RTL_QTR_OF_RTL_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "92", + "name" : "RTL_WEEK_OF_RTL_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "93", + "name" : "SEASON_OF_YEAR_ID", + "datatype" : "tinyint" + }, { + "id" : "94", + "name" : "YTM_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "95", + "name" : "YTQ_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "96", + "name" : "YTW_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "97", + "name" : "CRE_DATE", + "datatype" : "string" + }, { + "id" : "98", + "name" : "CRE_USER", + "datatype" : "string" + }, { + "id" : "99", + "name" : "UPD_DATE", + "datatype" : "string" + }, { + "id" : "100", + "name" : "UPD_USER", + "datatype" : "string" + } ], + "database" : "edw", + "last_modified" : 0 +} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json new file mode 100644 index 0000000..42f4d9c --- /dev/null +++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json @@ -0,0 +1,43 @@ +{ + "uuid" : "9ecc90c4-55df-436f-8602-2fbd4bca72e1", + "name" : "TEST_SELLER_TYPE_DIM", + "columns" : [ { + "id" : "1", + "name" : "SELLER_TYPE_CD", + "datatype" : "smallint" + }, { + "id" : "2", + "name" : "SELLER_TYPE_DESC", + "datatype" : "string" + }, { + "id" : "3", + "name" : "GLBL_RPRT_SLR_SGMNT_CD", + "datatype" : "tinyint" + }, { + "id" : "4", + "name" : "SELLER_GROUP_CD", + "datatype" : "tinyint" + }, { + "id" : "5", + "name" : "SELLER_GROUP_DESC", + "datatype" : "string" + }, { + "id" : "6", + "name" : "CRE_DATE", + "datatype" : "string" + }, { + "id" : "7", + "name" : "CRE_USER", + "datatype" : "string" + }, { + "id" : "8", + "name" : "UPD_DATE", + "datatype" : "string" + }, { + "id" : "9", + "name" : "UPD_USER", + "datatype" : "string" + } ], + "database" : "edw", + "last_modified" : 0 +} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json new file mode 100644 index 0000000..d451e34 --- /dev/null +++ b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json @@ -0,0 +1,47 @@ +{ + "uuid" : "338a3325-a947-46d1-9ece-e079b3b8d4a6", + "name" : "TEST_SITES", + "columns" : [ { + "id" : "1", + "name" : "SITE_ID", + "datatype" : "int" + }, { + "id" : "2", + "name" : "SITE_NAME", + "datatype" : "string" + }, { + "id" : "3", + "name" : "SITE_DOMAIN_CODE", + "datatype" : "string" + }, { + "id" : "4", + "name" : "DFAULT_LSTG_CURNCY", + "datatype" : "int" + }, { + "id" : "5", + "name" : "EOA_EMAIL_CSTMZBL_SITE_YN_ID", + "datatype" : "tinyint" + }, { + "id" : "6", + "name" : "SITE_CNTRY_ID", + "datatype" : "int" + }, { + "id" : "7", + "name" : "CRE_DATE", + "datatype" : "string" + }, { + "id" : "8", + "name" : "UPD_DATE", + "datatype" : "string" + }, { + "id" : "9", + "name" : "CRE_USER", + "datatype" : "string" + }, { + "id" : "10", + "name" : "UPD_USER", + "datatype" : "string" + } ], + "database" : "edw", + "last_modified" : 0 +} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/table/TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/TEST_CAL_DT.json deleted file mode 100644 index 0ad7ee1..0000000 --- a/examples/test_case_data/localmeta/table/TEST_CAL_DT.json +++ /dev/null @@ -1,407 +0,0 @@ -{ - "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a", - "name" : "TEST_CAL_DT", - "columns" : [ { - "id" : "1", - "name" : "CAL_DT", - "datatype" : "date" - }, { - "id" : "2", - "name" : "YEAR_BEG_DT", - "datatype" : "date" - }, { - "id" : "3", - "name" : "QTR_BEG_DT", - "datatype" : "date" - }, { - "id" : "4", - "name" : "MONTH_BEG_DT", - "datatype" : "date" - }, { - "id" : "5", - "name" : "WEEK_BEG_DT", - "datatype" : "date" - }, { - "id" : "6", - "name" : "AGE_FOR_YEAR_ID", - "datatype" : "smallint" - }, { - "id" : "7", - "name" : "AGE_FOR_QTR_ID", - "datatype" : "smallint" - }, { - "id" : "8", - "name" : "AGE_FOR_MONTH_ID", - "datatype" : "smallint" - }, { - "id" : "9", - "name" : "AGE_FOR_WEEK_ID", - "datatype" : "smallint" - }, { - "id" : "10", - "name" : "AGE_FOR_DT_ID", - "datatype" : "smallint" - }, { - "id" : "11", - "name" : "AGE_FOR_RTL_YEAR_ID", - "datatype" : "smallint" - }, { - "id" : "12", - "name" : "AGE_FOR_RTL_QTR_ID", - "datatype" : "smallint" - }, { - "id" : "13", - "name" : "AGE_FOR_RTL_MONTH_ID", - "datatype" : "smallint" - }, { - "id" : "14", - "name" : "AGE_FOR_RTL_WEEK_ID", - "datatype" : "smallint" - }, { - "id" : "15", - "name" : "AGE_FOR_CS_WEEK_ID", - "datatype" : "smallint" - }, { - "id" : "16", - "name" : "DAY_OF_CAL_ID", - "datatype" : "int" - }, { - "id" : "17", - "name" : "DAY_OF_YEAR_ID", - "datatype" : "smallint" - }, { - "id" : "18", - "name" : "DAY_OF_QTR_ID", - "datatype" : "smallint" - }, { - "id" : "19", - "name" : "DAY_OF_MONTH_ID", - "datatype" : "smallint" - }, { - "id" : "20", - "name" : "DAY_OF_WEEK_ID", - "datatype" : "int" - }, { - "id" : "21", - "name" : "WEEK_OF_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "22", - "name" : "WEEK_OF_CAL_ID", - "datatype" : "int" - }, { - "id" : "23", - "name" : "MONTH_OF_QTR_ID", - "datatype" : "tinyint" - }, { - "id" : "24", - "name" : "MONTH_OF_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "25", - "name" : "MONTH_OF_CAL_ID", - "datatype" : "smallint" - }, { - "id" : "26", - "name" : "QTR_OF_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "27", - "name" : "QTR_OF_CAL_ID", - "datatype" : "smallint" - }, { - "id" : "28", - "name" : "YEAR_OF_CAL_ID", - "datatype" : "smallint" - }, { - "id" : "29", - "name" : "YEAR_END_DT", - "datatype" : "string" - }, { - "id" : "30", - "name" : "QTR_END_DT", - "datatype" : "string" - }, { - "id" : "31", - "name" : "MONTH_END_DT", - "datatype" : "string" - }, { - "id" : "32", - "name" : "WEEK_END_DT", - "datatype" : "string" - }, { - "id" : "33", - "name" : "CAL_DT_NAME", - "datatype" : "string" - }, { - "id" : "34", - "name" : "CAL_DT_DESC", - "datatype" : "string" - }, { - "id" : "35", - "name" : "CAL_DT_SHORT_NAME", - "datatype" : "string" - }, { - "id" : "36", - "name" : "YTD_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "37", - "name" : "QTD_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "38", - "name" : "MTD_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "39", - "name" : "WTD_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "40", - "name" : "SEASON_BEG_DT", - "datatype" : "string" - }, { - "id" : "41", - "name" : "DAY_IN_YEAR_COUNT", - "datatype" : "smallint" - }, { - "id" : "42", - "name" : "DAY_IN_QTR_COUNT", - "datatype" : "tinyint" - }, { - "id" : "43", - "name" : "DAY_IN_MONTH_COUNT", - "datatype" : "tinyint" - }, { - "id" : "44", - "name" : "DAY_IN_WEEK_COUNT", - "datatype" : "tinyint" - }, { - "id" : "45", - "name" : "RTL_YEAR_BEG_DT", - "datatype" : "string" - }, { - "id" : "46", - "name" : "RTL_QTR_BEG_DT", - "datatype" : "string" - }, { - "id" : "47", - "name" : "RTL_MONTH_BEG_DT", - "datatype" : "string" - }, { - "id" : "48", - "name" : "RTL_WEEK_BEG_DT", - "datatype" : "string" - }, { - "id" : "49", - "name" : "CS_WEEK_BEG_DT", - "datatype" : "string" - }, { - "id" : "50", - "name" : "CAL_DATE", - "datatype" : "string" - }, { - "id" : "51", - "name" : "DAY_OF_WEEK", - "datatype" : "string" - }, { - "id" : "52", - "name" : "MONTH_ID", - "datatype" : "string" - }, { - "id" : "53", - "name" : "PRD_DESC", - "datatype" : "string" - }, { - "id" : "54", - "name" : "PRD_FLAG", - "datatype" : "string" - }, { - "id" : "55", - "name" : "PRD_ID", - "datatype" : "string" - }, { - "id" : "56", - "name" : "PRD_IND", - "datatype" : "string" - }, { - "id" : "57", - "name" : "QTR_DESC", - "datatype" : "string" - }, { - "id" : "58", - "name" : "QTR_ID", - "datatype" : "string" - }, { - "id" : "59", - "name" : "QTR_IND", - "datatype" : "string" - }, { - "id" : "60", - "name" : "RETAIL_WEEK", - "datatype" : "string" - }, { - "id" : "61", - "name" : "RETAIL_YEAR", - "datatype" : "string" - }, { - "id" : "62", - "name" : "RETAIL_START_DATE", - "datatype" : "string" - }, { - "id" : "63", - "name" : "RETAIL_WK_END_DATE", - "datatype" : "string" - }, { - "id" : "64", - "name" : "WEEK_IND", - "datatype" : "string" - }, { - "id" : "65", - "name" : "WEEK_NUM_DESC", - "datatype" : "string" - }, { - "id" : "66", - "name" : "WEEK_BEG_DATE", - "datatype" : "string" - }, { - "id" : "67", - "name" : "WEEK_END_DATE", - "datatype" : "string" - }, { - "id" : "68", - "name" : "WEEK_IN_YEAR_ID", - "datatype" : "string" - }, { - "id" : "69", - "name" : "WEEK_ID", - "datatype" : "string" - }, { - "id" : "70", - "name" : "WEEK_BEG_END_DESC_MDY", - "datatype" : "string" - }, { - "id" : "71", - "name" : "WEEK_BEG_END_DESC_MD", - "datatype" : "string" - }, { - "id" : "72", - "name" : "YEAR_ID", - "datatype" : "string" - }, { - "id" : "73", - "name" : "YEAR_IND", - "datatype" : "string" - }, { - "id" : "74", - "name" : "CAL_DT_MNS_1YEAR_DT", - "datatype" : "string" - }, { - "id" : "75", - "name" : "CAL_DT_MNS_2YEAR_DT", - "datatype" : "string" - }, { - "id" : "76", - "name" : "CAL_DT_MNS_1QTR_DT", - "datatype" : "string" - }, { - "id" : "77", - "name" : "CAL_DT_MNS_2QTR_DT", - "datatype" : "string" - }, { - "id" : "78", - "name" : "CAL_DT_MNS_1MONTH_DT", - "datatype" : "string" - }, { - "id" : "79", - "name" : "CAL_DT_MNS_2MONTH_DT", - "datatype" : "string" - }, { - "id" : "80", - "name" : "CAL_DT_MNS_1WEEK_DT", - "datatype" : "string" - }, { - "id" : "81", - "name" : "CAL_DT_MNS_2WEEK_DT", - "datatype" : "string" - }, { - "id" : "82", - "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "83", - "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "84", - "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "85", - "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "86", - "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "87", - "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "88", - "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND", - "datatype" : "tinyint" - }, { - "id" : "89", - "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND", - "datatype" : "tinyint" - }, { - "id" : "90", - "name" : "RTL_MONTH_OF_RTL_YEAR_ID", - "datatype" : "string" - }, { - "id" : "91", - "name" : "RTL_QTR_OF_RTL_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "92", - "name" : "RTL_WEEK_OF_RTL_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "93", - "name" : "SEASON_OF_YEAR_ID", - "datatype" : "tinyint" - }, { - "id" : "94", - "name" : "YTM_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "95", - "name" : "YTQ_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "96", - "name" : "YTW_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "97", - "name" : "CRE_DATE", - "datatype" : "string" - }, { - "id" : "98", - "name" : "CRE_USER", - "datatype" : "string" - }, { - "id" : "99", - "name" : "UPD_DATE", - "datatype" : "string" - }, { - "id" : "100", - "name" : "UPD_USER", - "datatype" : "string" - } ], - "database" : "edw", - "last_modified" : 0 -} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/table/TEST_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/TEST_CATEGORY_GROUPINGS.json deleted file mode 100644 index 7bcd092..0000000 --- a/examples/test_case_data/localmeta/table/TEST_CATEGORY_GROUPINGS.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f", - "name" : "TEST_CATEGORY_GROUPINGS", - "columns" : [ { - "id" : "1", - "name" : "LEAF_CATEG_ID", - "datatype" : "bigint" - }, { - "id" : "2", - "name" : "LEAF_CATEG_NAME", - "datatype" : "string" - }, { - "id" : "3", - "name" : "SITE_ID", - "datatype" : "int" - }, { - "id" : "4", - "name" : "CATEG_BUSN_MGR", - "datatype" : "string" - }, { - "id" : "5", - "name" : "CATEG_BUSN_UNIT", - "datatype" : "string" - }, { - "id" : "6", - "name" : "REGN_CATEG", - "datatype" : "string" - }, { - "id" : "7", - "name" : "USER_DEFINED_FIELD1", - "datatype" : "string" - }, { - "id" : "8", - "name" : "USER_DEFINED_FIELD3", - "datatype" : "string" - }, { - "id" : "9", - "name" : "CRE_DATE", - "datatype" : "string" - }, { - "id" : "10", - "name" : "UPD_DATE", - "datatype" : "string" - }, { - "id" : "11", - "name" : "CRE_USER", - "datatype" : "string" - }, { - "id" : "12", - "name" : "UPD_USER", - "datatype" : "string" - }, { - "id" : "13", - "name" : "META_CATEG_ID", - "datatype" : "decimal" - }, { - "id" : "14", - "name" : "META_CATEG_NAME", - "datatype" : "string" - }, { - "id" : "15", - "name" : "CATEG_LVL2_ID", - "datatype" : "decimal" - }, { - "id" : "16", - "name" : "CATEG_LVL3_ID", - "datatype" : "decimal" - }, { - "id" : "17", - "name" : "CATEG_LVL4_ID", - "datatype" : "decimal" - }, { - "id" : "18", - "name" : "CATEG_LVL5_ID", - "datatype" : "decimal" - }, { - "id" : "19", - "name" : "CATEG_LVL6_ID", - "datatype" : "decimal" - }, { - "id" : "20", - "name" : "CATEG_LVL7_ID", - "datatype" : "decimal" - }, { - "id" : "21", - "name" : "CATEG_LVL2_NAME", - "datatype" : "string" - }, { - "id" : "22", - "name" : "CATEG_LVL3_NAME", - "datatype" : "string" - }, { - "id" : "23", - "name" : "CATEG_LVL4_NAME", - "datatype" : "string" - }, { - "id" : "24", - "name" : "CATEG_LVL5_NAME", - "datatype" : "string" - }, { - "id" : "25", - "name" : "CATEG_LVL6_NAME", - "datatype" : "string" - }, { - "id" : "26", - "name" : "CATEG_LVL7_NAME", - "datatype" : "string" - }, { - "id" : "27", - "name" : "CATEG_FLAGS", - "datatype" : "decimal" - }, { - "id" : "28", - "name" : "ADULT_CATEG_YN", - "datatype" : "string" - }, { - "id" : "29", - "name" : "DOMAIN_ID", - "datatype" : "decimal" - }, { - "id" : "30", - "name" : "USER_DEFINED_FIELD5", - "datatype" : "string" - }, { - "id" : "31", - "name" : "VCS_ID", - "datatype" : "decimal" - }, { - "id" : "32", - "name" : "GCS_ID", - "datatype" : "decimal" - }, { - "id" : "33", - "name" : "MOVE_TO", - "datatype" : "decimal" - }, { - "id" : "34", - "name" : "SAP_CATEGORY_ID", - "datatype" : "decimal" - }, { - "id" : "35", - "name" : "SRC_ID", - "datatype" : "tinyint" - }, { - "id" : "36", - "name" : "BSNS_VRTCL_NAME", - "datatype" : "string" - } ], - "database" : "DEFAULT", - "last_modified" : 0 -} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/table/TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/TEST_KYLIN_FACT.json deleted file mode 100644 index 732351a..0000000 --- a/examples/test_case_data/localmeta/table/TEST_KYLIN_FACT.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771", - "name" : "TEST_KYLIN_FACT", - "columns" : [ { - "id" : "1", - "name" : "TRANS_ID", - "datatype" : "bigint" - }, { - "id" : "2", - "name" : "CAL_DT", - "datatype" : "date" - }, { - "id" : "3", - "name" : "LSTG_FORMAT_NAME", - "datatype" : "string" - }, { - "id" : "4", - "name" : "LEAF_CATEG_ID", - "datatype" : "bigint" - }, { - "id" : "5", - "name" : "LSTG_SITE_ID", - "datatype" : "int" - }, { - "id" : "6", - "name" : "SLR_SEGMENT_CD", - "datatype" : "smallint" - }, { - "id" : "7", - "name" : "PRICE", - "datatype" : "decimal(18,6)" - }, { - "id" : "8", - "name" : "ITEM_COUNT", - "datatype" : "bigint" - }, { - "id" : "9", - "name" : "SELLER_ID", - "datatype" : "bigint" - } ], - "database" : "DEFAULT", - "last_modified" : 0 -} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/table/TEST_SELLER_TYPE_DIM.json b/examples/test_case_data/localmeta/table/TEST_SELLER_TYPE_DIM.json deleted file mode 100644 index 42f4d9c..0000000 --- a/examples/test_case_data/localmeta/table/TEST_SELLER_TYPE_DIM.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "uuid" : "9ecc90c4-55df-436f-8602-2fbd4bca72e1", - "name" : "TEST_SELLER_TYPE_DIM", - "columns" : [ { - "id" : "1", - "name" : "SELLER_TYPE_CD", - "datatype" : "smallint" - }, { - "id" : "2", - "name" : "SELLER_TYPE_DESC", - "datatype" : "string" - }, { - "id" : "3", - "name" : "GLBL_RPRT_SLR_SGMNT_CD", - "datatype" : "tinyint" - }, { - "id" : "4", - "name" : "SELLER_GROUP_CD", - "datatype" : "tinyint" - }, { - "id" : "5", - "name" : "SELLER_GROUP_DESC", - "datatype" : "string" - }, { - "id" : "6", - "name" : "CRE_DATE", - "datatype" : "string" - }, { - "id" : "7", - "name" : "CRE_USER", - "datatype" : "string" - }, { - "id" : "8", - "name" : "UPD_DATE", - "datatype" : "string" - }, { - "id" : "9", - "name" : "UPD_USER", - "datatype" : "string" - } ], - "database" : "edw", - "last_modified" : 0 -} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/table/TEST_SITES.json b/examples/test_case_data/localmeta/table/TEST_SITES.json deleted file mode 100644 index d451e34..0000000 --- a/examples/test_case_data/localmeta/table/TEST_SITES.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "uuid" : "338a3325-a947-46d1-9ece-e079b3b8d4a6", - "name" : "TEST_SITES", - "columns" : [ { - "id" : "1", - "name" : "SITE_ID", - "datatype" : "int" - }, { - "id" : "2", - "name" : "SITE_NAME", - "datatype" : "string" - }, { - "id" : "3", - "name" : "SITE_DOMAIN_CODE", - "datatype" : "string" - }, { - "id" : "4", - "name" : "DFAULT_LSTG_CURNCY", - "datatype" : "int" - }, { - "id" : "5", - "name" : "EOA_EMAIL_CSTMZBL_SITE_YN_ID", - "datatype" : "tinyint" - }, { - "id" : "6", - "name" : "SITE_CNTRY_ID", - "datatype" : "int" - }, { - "id" : "7", - "name" : "CRE_DATE", - "datatype" : "string" - }, { - "id" : "8", - "name" : "UPD_DATE", - "datatype" : "string" - }, { - "id" : "9", - "name" : "CRE_USER", - "datatype" : "string" - }, { - "id" : "10", - "name" : "UPD_USER", - "datatype" : "string" - } ], - "database" : "edw", - "last_modified" : 0 -} \ No newline at end of file diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java index b0f06ad..af4909d 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java +++ b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java @@ -85,7 +85,6 @@ public TblColRef findPKByFK(TblColRef fk) { public void init(Map tables) { this.errors.clear(); - this.factTable = factTable.toUpperCase(); initJoinColumns(tables); } @@ -95,7 +94,7 @@ private void initJoinColumns(Map tables) { // initDimensionColumns() will do the update for (LookupDesc lookup : this.lookups) { lookup.setTable(lookup.getTable().toUpperCase()); - TableDesc dimTable = tables.get(lookup.getTable()); + TableDesc dimTable = tables.get(TableDesc.getTableIdentity(lookup.getTable())); JoinDesc join = lookup.getJoin(); if (join == null) @@ -117,7 +116,7 @@ private void initJoinColumns(Map tables) { } join.setPrimaryKeyColumns(pkCols); // foreign key - TableDesc factTable = tables.get(this.getFactTable()); + TableDesc factTable = tables.get(TableDesc.getTableIdentity(this.factTable)); if (factTable == null) { addError("Fact table does not exist:" + this.getFactTable()); }