From 5259307fea2d9a5f4022ca6214510cdc5ca52c1e Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Thu, 4 Dec 2014 19:42:42 +0800 Subject: [PATCH] fix the issues after adding database name to ahead of table name; --- .../java/com/kylinolap/cube/model/CubeDesc.java | 59 ++++++++++++---------- .../kylinolap/cube/model/CubePartitionDesc.java | 11 ++-- .../com/kylinolap/cube/model/DimensionDesc.java | 34 ++++++++----- .../java/com/kylinolap/dict/LookupTableTest.java | 8 +-- .../com/kylinolap/dict/SnapshotManagerTest.java | 2 +- .../localmeta/cube/test_kylin_cube_ii.json | 39 -------------- .../java/com/kylinolap/job/JoinedFlatTable.java | 21 ++++++-- .../job/hadoop/hive/JoinedFlatTableDesc.java | 32 ++++++++++++ .../test/java/com/kylinolap/job/DeployUtil.java | 11 ++-- .../kylinolap/metadata/MetadataManagerTest.java | 4 +- 10 files changed, 120 insertions(+), 101 deletions(-) delete mode 100644 examples/test_case_data/localmeta/cube/test_kylin_cube_ii.json diff --git a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java index dda39d2..d728b2d 100644 --- a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java +++ b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java @@ -94,7 +94,7 @@ public String toString() { private KylinConfig config; private DataModelDesc model; - + @JsonProperty("name") private String name; @JsonProperty("model_name") @@ -237,9 +237,9 @@ public DimensionDesc findDimensionByName(String dimName) { return result; } -// public boolean isFactTable(String factTable) { -// return this.factTable.equalsIgnoreCase(factTable); -// } + // public boolean isFactTable(String factTable) { + // return this.factTable.equalsIgnoreCase(factTable); + // } public boolean isDerived(TblColRef col) { return derivedToHostMap.containsKey(col); @@ -459,45 +459,44 @@ public String calculateSignature() { public void init(KylinConfig config, Map tables) { this.errors.clear(); this.config = config; - if(this.modelName == null || this.modelName.length() ==0) { + if (this.modelName == null || this.modelName.length() == 0) { this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); } - + this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); - - if(this.model == null) { + + if (this.model == null) { this.addError("No data model found with name '" + modelName + "'."); } - + //key: column name; value: list of tables; Map> columnTableMap = new HashMap>(); - + String colName; - for(TableDesc table : tables.values()) { - for(ColumnDesc col : table.getColumns()) { + for (TableDesc table : tables.values()) { + for (ColumnDesc col : table.getColumns()) { colName = col.getName(); List tableNames = columnTableMap.get(colName); - if(tableNames == null) { + if (tableNames == null) { tableNames = new LinkedList(); columnTableMap.put(colName, tableNames); - } + } tableNames.add(table); } } - // key: table name; value: list of databases; Map> tableDatabaseMap = new HashMap>(); - + String tableName; - for(TableDesc table : tables.values()) { + for (TableDesc table : tables.values()) { tableName = table.getName(); - List dbNames = tableDatabaseMap.get(tableName); - if(dbNames == null) { - dbNames = new LinkedList(); - tableDatabaseMap.put(tableName, dbNames); - } - dbNames.add(table.getDatabase()); + List dbNames = tableDatabaseMap.get(tableName); + if (dbNames == null) { + dbNames = new LinkedList(); + tableDatabaseMap.put(tableName, dbNames); + } + dbNames.add(table.getDatabase()); } for (DimensionDesc dim : dimensions) { @@ -538,26 +537,32 @@ private void initDimensionColumns(Map tables) { // dimension column if (dim.getColumn() != null) { //if ("{FK}".equals(dim.getColumn())) { - if (join != null) { + if (join != null) { + // this dimension is defined on lookup table for (TblColRef ref : join.getForeignKeyColumns()) { TblColRef inited = initDimensionColRef(ref); dimColList.add(inited); hostColList.add(inited); } } else { + // this dimension is defined on fact table for (String aColumn : dim.getColumn()) { TblColRef ref = initDimensionColRef(dimTable, aColumn); - dimColList.add(ref); - hostColList.add(ref); + if (!dimColList.contains(ref)) { + dimColList.add(ref); + //hostColList.add(ref); + } } } } + // hierarchy columns if (dim.getHierarchy() != null) { for (HierarchyDesc hier : dim.getHierarchy()) { TblColRef ref = initDimensionColRef(dimTable, hier.getColumn()); hier.setColumnRef(ref); - dimColList.add(ref); + if (!dimColList.contains(ref)) + dimColList.add(ref); } if (hostColList.isEmpty()) { // the last hierarchy could serve // as host when col is diff --git a/cube/src/main/java/com/kylinolap/cube/model/CubePartitionDesc.java b/cube/src/main/java/com/kylinolap/cube/model/CubePartitionDesc.java index 4f174d2..bc3c680 100644 --- a/cube/src/main/java/com/kylinolap/cube/model/CubePartitionDesc.java +++ b/cube/src/main/java/com/kylinolap/cube/model/CubePartitionDesc.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import com.kylinolap.common.util.StringSplitter; import com.kylinolap.metadata.model.realization.TblColRef; /** @@ -45,12 +46,14 @@ public void init(Map> columnMap) { if (null != partitionDateColumn) { - String[] columns = partitionDateColumn.split("\\."); + partitionDateColumn = partitionDateColumn.toUpperCase(); + + String[] columns = StringSplitter.split(partitionDateColumn, "."); - if (null != columns && columns.length == 2) { - Map cols = columnMap.get(columns[0].toUpperCase()); + if (null != columns && columns.length == 3) { + Map cols = columnMap.get(columns[0].toUpperCase() + "." + columns[1].toUpperCase()); if (cols != null) - partitionDateColumnRef = cols.get(columns[1].toUpperCase()); + partitionDateColumnRef = cols.get(columns[2].toUpperCase()); } } diff --git a/cube/src/main/java/com/kylinolap/cube/model/DimensionDesc.java b/cube/src/main/java/com/kylinolap/cube/model/DimensionDesc.java index c1c3994..39e8fb1 100644 --- a/cube/src/main/java/com/kylinolap/cube/model/DimensionDesc.java +++ b/cube/src/main/java/com/kylinolap/cube/model/DimensionDesc.java @@ -69,9 +69,12 @@ public boolean isHierarchyColumn(TblColRef col) { } return false; } + + public boolean isHierarchy() { + return isHierarchy; + } /** - * @deprecated use getTableDesc() to get accurate table info * @return */ public String getTable() { @@ -196,7 +199,7 @@ public String toString() { if (tableName == null) { List tables = columnTableMap.get(thisColumn); if (tables == null) { - throw new IllegalStateException("The column '" + thisColumn + "' isn't appeared on any table."); + throw new IllegalStateException("The column '" + thisColumn + "' isn't appeared in any table."); } else if (tables.size() > 1) { throw new IllegalStateException("The column '" + thisColumn + "' is ambiguous; it appeared in more than one tables, please specify table name together with the column name."); } else { @@ -232,28 +235,31 @@ public void init(CubeDesc cubeDesc, Map tables, Map 0) { sql.append(","); } - sql.append(col.getTableName() + "." + col.getColumnName() + "\n"); + String tableAlias = intermediateTableDesc.getTableAlias(col.getTableName()); + sql.append(tableAlias + "." + col.getColumnName() + "\n"); } appendJoinStatement(intermediateTableDesc, sql); appendWhereStatement(intermediateTableDesc, sql); @@ -140,7 +141,8 @@ private static void appendJoinStatement(JoinedFlatTableDesc intermediateTableDes CubeDesc cubeDesc = intermediateTableDesc.getCubeDesc(); String factTableName = cubeDesc.getFactTable(); - sql.append("FROM " + factTableName + "\n"); + String factTableAlias = intermediateTableDesc.getTableAlias(factTableName); + sql.append("FROM " + factTableName + " as " + factTableAlias + " \n"); for (DimensionDesc dim : cubeDesc.getDimensions()) { JoinDesc join = dim.getJoin(); @@ -153,13 +155,13 @@ private static void appendJoinStatement(JoinedFlatTableDesc intermediateTableDes if (pk.length != fk.length) { throw new RuntimeException("Invalid join condition of dimension " + dim.getName()); } - sql.append(joinType + " JOIN " + dimTableName + "\n"); + sql.append(joinType + " JOIN " + dimTableName + " as " + intermediateTableDesc.getTableAlias(dimTableName) + "\n"); sql.append("ON "); for (int i = 0; i < pk.length; i++) { if (i > 0) { sql.append(" AND "); } - sql.append(factTableName + "." + fk[i].getName() + " = " + dimTableName + "." + pk[i].getName()); + sql.append(factTableAlias + "." + fk[i].getName() + " = " + intermediateTableDesc.getTableAlias(dimTableName) + "." + pk[i].getName()); } sql.append("\n"); @@ -192,7 +194,16 @@ private static void appendWhereStatement(JoinedFlatTableDesc intermediateTableDe } if (!(dateStart == 0 && dateEnd == 0)) { String partitionColumnName = cubeDesc.getCubePartitionDesc().getPartitionDateColumn(); - + int indexOfDot = partitionColumnName.lastIndexOf("."); + + // convert to use table alias; + if(indexOfDot >0) { + String partitionTableName = partitionColumnName.substring(0, indexOfDot); + String columeOnly = partitionColumnName.substring(indexOfDot); + String partitionTableAlias = intermediateTableDesc.getTableAlias(partitionTableName); + partitionColumnName = partitionTableAlias + columeOnly; + } + whereBuilder.append(hasCondition ? " AND (" : " ("); if (dateStart > 0) { whereBuilder.append(partitionColumnName + " >= '" + formatDateTimeInWhereClause(dateStart) + "' "); diff --git a/job/src/main/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableDesc.java b/job/src/main/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableDesc.java index 7356c70..56770a7 100644 --- a/job/src/main/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableDesc.java +++ b/job/src/main/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableDesc.java @@ -23,7 +23,9 @@ import com.kylinolap.cube.CubeSegment; import com.kylinolap.cube.cuboid.Cuboid; import com.kylinolap.cube.model.CubeDesc; +import com.kylinolap.cube.model.DimensionDesc; import com.kylinolap.cube.model.MeasureDesc; +import com.kylinolap.metadata.model.JoinDesc; import com.kylinolap.metadata.model.realization.FunctionDesc; import com.kylinolap.metadata.model.realization.TblColRef; @@ -39,6 +41,9 @@ private int[] rowKeyColumnIndexes; // the column index on flat table private int[][] measureColumnIndexes; // [i] is the i.th measure related // column index on flat table + + // Map for table alais; key: table name; value: alias; + private Map tableAliasMap; public JoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment) { this.cubeDesc = cubeDesc; @@ -112,6 +117,28 @@ private void parseCubeDesc() { } } } + + buileTableAliasMap(); + } + + private void buileTableAliasMap() { + tableAliasMap = new HashMap(); + + tableAliasMap.put(cubeDesc.getFactTable(), "FACT_TABLE"); + + int i=1; + for (DimensionDesc dim : cubeDesc.getDimensions()) { + JoinDesc join = dim.getJoin(); + if(join != null) { + tableAliasMap.put(dim.getTable(), "LOOKUP_" + i); + i++; + } + + } + } + + public String getTableAlias(String tableName) { + return tableAliasMap.get(tableName); } private int contains(List columnList, TblColRef c) { @@ -144,6 +171,7 @@ public String getTableName(String jobUUID) { private String columnName; private String dataType; private String tableName; + private String databaseName; public IntermediateColumnDesc(String id, String columnName, String dataType, String tableName) { this.id = id; @@ -167,5 +195,9 @@ public String getDataType() { public String getTableName() { return tableName; } + + public String getDatabaseName() { + return databaseName; + } } } diff --git a/job/src/test/java/com/kylinolap/job/DeployUtil.java b/job/src/test/java/com/kylinolap/job/DeployUtil.java index 65a4d9a..6fd59af 100644 --- a/job/src/test/java/com/kylinolap/job/DeployUtil.java +++ b/job/src/test/java/com/kylinolap/job/DeployUtil.java @@ -120,11 +120,11 @@ private static KylinConfig config() { // ============================================================================ - static final String TABLE_CAL_DT = "test_cal_dt"; - static final String TABLE_CATEGORY_GROUPINGS = "test_category_groupings"; - static final String TABLE_KYLIN_FACT = "test_kylin_fact"; - static final String TABLE_SELLER_TYPE_DIM = "test_seller_type_dim"; - static final String TABLE_SITES = "test_sites"; + static final String TABLE_CAL_DT = "edw.test_cal_dt"; + static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings"; + static final String TABLE_KYLIN_FACT = "default.test_kylin_fact"; + static final String TABLE_SELLER_TYPE_DIM = "edw.test_seller_type_dim"; + static final String TABLE_SITES = "edw.test_sites"; static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM, TABLE_SITES }; @@ -167,6 +167,7 @@ private static void deployHiveTables() throws Exception { temp.delete(); // create hive tables + execHiveCommand("CREATE DATABASE IF NOT EXISTS EDW;"); execHiveCommand(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase()))); execHiveCommand(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase()))); execHiveCommand(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase()))); diff --git a/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java b/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java index 63be707..ce273b5 100644 --- a/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java +++ b/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java @@ -50,9 +50,9 @@ public void testListAllTables() throws Exception { @Test public void testFindTableByName() throws Exception { - TableDesc table = MetadataManager.getInstance(this.getTestConfig()).getTableDesc("TEST_CAL_DT"); + TableDesc table = MetadataManager.getInstance(this.getTestConfig()).getTableDesc("EDW.TEST_CAL_DT"); Assert.assertNotNull(table); - Assert.assertEquals("TEST_CAL_DT", table.getIdentity()); + Assert.assertEquals("EDW.TEST_CAL_DT", table.getIdentity()); } @Test