From f231c2d7c29f4ecfd00e2ddcf347b3d2e2ca6497 Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Sun, 4 Jan 2015 16:20:40 +0800 Subject: [PATCH 1/2] Remove the default value for database name; Clients need provide the database name explicitly. --- .../kylinolap/cube/project/ProjectManagerTest.java | 10 ++++---- .../com/kylinolap/invertedindex/model/IIDesc.java | 3 +-- .../com/kylinolap/metadata/MetadataManager.java | 10 ++++---- .../kylinolap/metadata/model/DataModelDesc.java | 8 +++---- .../com/kylinolap/metadata/model/TableDesc.java | 24 ++----------------- .../com/kylinolap/metadata/model/TblColRef.java | 3 ++- .../kylinolap/metadata/project/ProjectManager.java | 7 +++--- .../kylinolap/metadata/model/TableDescTest.java | 28 ---------------------- .../storage/filter/BitMapFilterEvaluatorTest.java | 1 + .../storage/filter/FilterSerializeTest.java | 2 ++ .../observer/AggregateRegionObserverTest.java | 1 + 11 files changed, 26 insertions(+), 71 deletions(-) diff --git a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java b/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java index 84c1275..1d577ac 100644 --- a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java @@ -137,27 +137,27 @@ public void testExistingProject() throws Exception { @Test public void testProjectsDrop() throws IOException { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_empty"); - assertTrue(ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "test_kylin_fact").contains(cube)); + assertTrue(ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "default.test_kylin_fact").contains(cube)); assertTrue(ProjectManager.getInstance(this.getTestConfig()).listAllRealizations("default").contains(cube)); CubeManager.getInstance(getTestConfig()).dropCube(cube.getName(), true); - assertTrue(!ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "test_kylin_fact").contains(cube)); + assertTrue(!ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "default.test_kylin_fact").contains(cube)); assertTrue(!ProjectManager.getInstance(this.getTestConfig()).listAllRealizations("default").contains(cube)); } @Test public void testProjectsLoadAfterProjectChange() throws IOException { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_empty"); - assertTrue(ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "test_kylin_fact").contains(cube)); + assertTrue(ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "default.test_kylin_fact").contains(cube)); ProjectManager.getInstance(getTestConfig()).removeRealizationsFromProjects(RealizationType.CUBE, cube.getName()); - assertTrue(!ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "test_kylin_fact").contains(cube)); + assertTrue(!ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "default.test_kylin_fact").contains(cube)); ProjectManager.getInstance(getTestConfig()).updateRealizationToProject(RealizationType.CUBE, cube.getName(), "default", "tester"); - assertTrue(ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "test_kylin_fact").contains(cube)); + assertTrue(ProjectManager.getInstance(this.getTestConfig()).getRealizationsByTable("default", "default.test_kylin_fact").contains(cube)); } public CubeDescManager getCubeDescManager() { diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/model/IIDesc.java index 78961cd..9c1bf58 100644 --- a/invertedindex/src/main/java/com/kylinolap/invertedindex/model/IIDesc.java +++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/model/IIDesc.java @@ -88,7 +88,6 @@ public void init(MetadataManager metadataManager) { + modelName + "'."); } - factTableName = TableDesc.getTableIdentity(factTableName); timestampDimension = timestampDimension.toUpperCase(); // capitalize @@ -153,7 +152,7 @@ public void init(MetadataManager metadataManager) { for (int i = 0; i < allColumns.size(); ++i) { TblColRef col = allColumns.get(i); if (col.getTable().equalsIgnoreCase( - TableDesc.getTableIdentity(this.factTableName)) + this.factTableName) && col.getColumn().getName() .equalsIgnoreCase(this.timestampDimension)) { tsCol = i; diff --git a/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java b/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java index 98cc680..335e1db 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java +++ b/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java @@ -145,7 +145,7 @@ public ResourceStore getStore() { * @return */ public TableDesc getTableDesc(String tableName) { - return srcTableMap.get(TableDesc.getTableIdentity(tableName)); + return srcTableMap.get(tableName.toUpperCase()); } /** @@ -155,7 +155,7 @@ public TableDesc getTableDesc(String tableName) { * @return */ public Map getTableDescExd(String tableName) { - String tableIdentity = TableDesc.getTableIdentity(tableName); + String tableIdentity = tableName; Map result = new HashMap(); if (srcTableExdMap.containsKey(tableIdentity)) { Map tmp = srcTableExdMap.get(tableIdentity); @@ -175,7 +175,7 @@ public void createSourceTable(TableDesc srcTable) throws IOException { if (srcTable.getUuid() == null || srcTable.getIdentity() == null) { throw new IllegalArgumentException(); } - String tableIdentity = TableDesc.getTableIdentity(srcTable); + String tableIdentity = srcTable.getIdentity(); if (srcTableMap.containsKey(tableIdentity)) { throw new IllegalArgumentException("SourceTable '" + srcTable.getIdentity() + "' already exists"); } @@ -204,7 +204,7 @@ private void reloadAllSourceTableExd() throws IOException { for (String path : paths) { Map attrContainer = new HashMap(); String tableName = loadSourceTableExd(getStore(), path, attrContainer); - String tableIdentity = TableDesc.getTableIdentity(tableName); + String tableIdentity = tableName; checkNoDupName(tableIdentity, srcTableExdMap.containsKey(tableIdentity), "SourceTableExd", path); srcTableExdMap.putLocal(tableIdentity, attrContainer); @@ -241,7 +241,7 @@ private void reloadAllSourceTable() throws IOException { List paths = store.collectResourceRecursively(ResourceStore.TABLE_RESOURCE_ROOT, MetadataConstances.FILE_SURFIX); for (String path : paths) { TableDesc t = loadSourceTable(path); - String tableIdentity = TableDesc.getTableIdentity(t); + String tableIdentity = t.getIdentity(); checkNoDupName(tableIdentity, srcTableMap.containsKey(tableIdentity), "SourceTable", path); srcTableMap.putLocal(tableIdentity, t); diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java index 7b16415..be98c35 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java +++ b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java @@ -48,11 +48,11 @@ public void setName(String name) { public String getFactTable() { - return TableDesc.getTableIdentity(factTable); + return factTable; } public void setFactTable(String factTable) { - this.factTable = factTable; + this.factTable = factTable.toUpperCase(); } public LookupDesc[] getLookups() { @@ -100,7 +100,7 @@ private void initJoinColumns(Map tables) { // initDimensionColumns() will do the update for (LookupDesc lookup : this.lookups) { lookup.setTable(lookup.getTable().toUpperCase()); - TableDesc dimTable = tables.get(TableDesc.getTableIdentity(lookup.getTable())); + TableDesc dimTable = tables.get(lookup.getTable()); JoinDesc join = lookup.getJoin(); if (join == null) @@ -122,7 +122,7 @@ private void initJoinColumns(Map tables) { } join.setPrimaryKeyColumns(pkCols); // foreign key - TableDesc factTable = tables.get(TableDesc.getTableIdentity(this.factTable)); + TableDesc factTable = tables.get(this.factTable.toUpperCase()); if (factTable == null) { addError("Fact table does not exist:" + this.getFactTable()); } diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/TableDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/TableDesc.java index 0d481b8..b42de98 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/model/TableDesc.java +++ b/metadata/src/main/java/com/kylinolap/metadata/model/TableDesc.java @@ -58,11 +58,11 @@ public ColumnDesc findColumnByName(String name) { } public String getResourcePath() { - return ResourceStore.TABLE_RESOURCE_ROOT + "/" + getTableIdentity(this) + ".json"; + return ResourceStore.TABLE_RESOURCE_ROOT + "/" + getIdentity() + ".json"; } public String getIdentity() { - return TableDesc.getTableIdentity(this); + return String.format("%s.%s", this.getDatabase().toUpperCase(), this.getName()).toUpperCase(); } // ============================================================================ @@ -87,9 +87,6 @@ public void setName(String name) { @JsonProperty("database") public String getDatabase() { - if (database == null) { - return "DEFAULT"; - } return database.getName(); } @@ -147,21 +144,4 @@ public int compare(ColumnDesc col1, ColumnDesc col2) { public String toString() { return "TableDesc [database=" + getDatabase() + " name=" + name + "]"; } - - private static final Pattern TABLE_IDENTITY_PATTERN = Pattern.compile("^\\w+\\.\\w+$"); - - public static String getTableIdentity(TableDesc tableDesc) { - return String.format("%s.%s", tableDesc.getDatabase(), tableDesc.getName()).toUpperCase(); - } - - public static String getTableIdentity(String tableName) { - if (!tableName.contains(".")) { - tableName = "DEFAULT." + tableName; - } - if (!TABLE_IDENTITY_PATTERN.matcher(tableName).matches()) { - throw new IllegalArgumentException("invalid tableName:" + tableName); - } - return tableName.toUpperCase(); - } - } diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/TblColRef.java b/metadata/src/main/java/com/kylinolap/metadata/model/TblColRef.java index 96b5fce..35dfd2c 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/model/TblColRef.java +++ b/metadata/src/main/java/com/kylinolap/metadata/model/TblColRef.java @@ -98,6 +98,7 @@ public DataType getType() { public void markInnerColumn(InnerDataTypeEnum dataType) { this.column.setDatatype(dataType.getDataType()); this.column.getTable().setName(INNER_TABLE_NAME); + this.column.getTable().setDatabase("DEFAULT"); } public boolean isInnerColumn() { @@ -137,7 +138,7 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; TblColRef other = (TblColRef) obj; - if (!StringUtils.equals(TableDesc.getTableIdentity(column.getTable()), TableDesc.getTableIdentity(other.column.getTable()))) + if (!StringUtils.equals(column.getTable().getIdentity(), other.column.getTable().getIdentity())) return false; if (!StringUtils.equals(column.getName(), other.column.getName())) return false; diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java index 57d1532..9be8ec2 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java @@ -502,22 +502,21 @@ private ProjectTable getProjectTable(String project, final String table) { } private ProjectTable getProjectTable(String project, final String table, boolean autoCreate) { - String tableIdentity = TableDesc.getTableIdentity(table); ProjectTable projectTable = null; project = ProjectInstance.getNormalizedProjectName(project); - if (this.projectTables.containsEntry(project, new ProjectTable(tableIdentity))) { + if (this.projectTables.containsEntry(project, new ProjectTable(table))) { Iterator projsIter = this.projectTables.get(project).iterator(); while (projsIter.hasNext()) { ProjectTable oneTable = projsIter.next(); - if (oneTable.getName().equalsIgnoreCase(tableIdentity)) { + if (oneTable.getName().equalsIgnoreCase(table)) { projectTable = oneTable; break; } } } else { - projectTable = new ProjectTable(tableIdentity); + projectTable = new ProjectTable(table); if (autoCreate) { this.projectTables.put(project, projectTable); diff --git a/metadata/src/test/java/com/kylinolap/metadata/model/TableDescTest.java b/metadata/src/test/java/com/kylinolap/metadata/model/TableDescTest.java index d8a0955..07891d3 100644 --- a/metadata/src/test/java/com/kylinolap/metadata/model/TableDescTest.java +++ b/metadata/src/test/java/com/kylinolap/metadata/model/TableDescTest.java @@ -10,32 +10,4 @@ */ public class TableDescTest { - @Test - public void testTableIdentity() { - final String tableName = "testtable"; - final String dbName = "testdb"; - TableDesc table = new TableDesc(); - table.setName(tableName); - assertEquals(("DEFAULT." + tableName).toUpperCase(), TableDesc.getTableIdentity(table)); - assertEquals(("DEFAULT." + tableName).toUpperCase(), TableDesc.getTableIdentity(tableName)); - assertEquals(("DEFAULT." + tableName).toUpperCase(), TableDesc.getTableIdentity("DEFAULT." + tableName)); - assertEquals((dbName + "." + tableName).toUpperCase(), TableDesc.getTableIdentity(dbName + "." + tableName)); - - table.setDatabase(dbName); - assertEquals((dbName + "." + tableName).toUpperCase(), TableDesc.getTableIdentity(dbName + "." + tableName)); - - try { - TableDesc.getTableIdentity("1 2"); - fail("should throw IllegalArgumentException"); - } catch (IllegalArgumentException ex) { - System.out.println(ex.getMessage()); - } - try { - TableDesc.getTableIdentity("t.2.abc"); - fail("should throw IllegalArgumentException"); - } catch (IllegalArgumentException ex) { - System.out.println(ex.getMessage()); - } - - } } diff --git a/storage/src/test/java/com/kylinolap/storage/filter/BitMapFilterEvaluatorTest.java b/storage/src/test/java/com/kylinolap/storage/filter/BitMapFilterEvaluatorTest.java index 86f1827..fedba0b 100644 --- a/storage/src/test/java/com/kylinolap/storage/filter/BitMapFilterEvaluatorTest.java +++ b/storage/src/test/java/com/kylinolap/storage/filter/BitMapFilterEvaluatorTest.java @@ -24,6 +24,7 @@ static { TableDesc table = new TableDesc(); table.setName("TABLE"); + table.setDatabase("DEFAULT"); ColumnDesc col = new ColumnDesc(); col.setTable(table); diff --git a/storage/src/test/java/com/kylinolap/storage/filter/FilterSerializeTest.java b/storage/src/test/java/com/kylinolap/storage/filter/FilterSerializeTest.java index 3f3f0a0..c948dbd 100644 --- a/storage/src/test/java/com/kylinolap/storage/filter/FilterSerializeTest.java +++ b/storage/src/test/java/com/kylinolap/storage/filter/FilterSerializeTest.java @@ -110,6 +110,7 @@ public void testSerialize06() { public void testSerialize07() { TableDesc table = new TableDesc(); table.setName("TEST_KYLIN_FACT"); + table.setDatabase("DEFAULT"); ColumnDesc column = new ColumnDesc(); column.setTable(table); @@ -127,6 +128,7 @@ public void testSerialize07() { @Test public void testSerialize08() { TableDesc table = new TableDesc(); + table.setDatabase("DEFAULT"); ColumnDesc column = new ColumnDesc(); column.setTable(table); diff --git a/storage/src/test/java/com/kylinolap/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java b/storage/src/test/java/com/kylinolap/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java index 76b0389..a6597ff 100644 --- a/storage/src/test/java/com/kylinolap/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java +++ b/storage/src/test/java/com/kylinolap/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java @@ -199,6 +199,7 @@ private String toString(byte[] array, int offset, short length, byte[] mask) { private CoprocessorRowType newRowType() { TableDesc t = new TableDesc(); t.setName("TABLE"); + t.setDatabase("DEFAULT"); TblColRef[] cols = new TblColRef[] { newCol("A", t), newCol("B", t), newCol("C", t), newCol("D", t) }; int[] sizes = new int[] { 1, 1, 1, 1 }; return new CoprocessorRowType(cols, sizes); From f323017051c156795f6a0204901ab9f0d93f83aa Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Sun, 4 Jan 2015 17:08:18 +0800 Subject: [PATCH 2/2] Update some test cases --- dictionary/src/test/java/com/kylinolap/dict/NumberDictionaryTest.java | 4 ++++ .../test/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableTest.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dictionary/src/test/java/com/kylinolap/dict/NumberDictionaryTest.java b/dictionary/src/test/java/com/kylinolap/dict/NumberDictionaryTest.java index 8343877..53efd6a 100644 --- a/dictionary/src/test/java/com/kylinolap/dict/NumberDictionaryTest.java +++ b/dictionary/src/test/java/com/kylinolap/dict/NumberDictionaryTest.java @@ -90,6 +90,10 @@ public void testDictionary() { for (int i = 0; i < sorted.size(); i++) { String dictNum = dict.getValueFromId(i); System.out.println(sorted.get(i) + "\t" + dictNum); + } + + for (int i = 0; i < sorted.size(); i++) { + String dictNum = dict.getValueFromId(i); assertEquals(sorted.get(i), new BigDecimal(dictNum)); } diff --git a/job/src/test/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableTest.java b/job/src/test/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableTest.java index c35b377..9dbf6ba 100644 --- a/job/src/test/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableTest.java +++ b/job/src/test/java/com/kylinolap/job/hadoop/hive/JoinedFlatTableTest.java @@ -75,7 +75,7 @@ public void testGenerateInsertSql() throws IOException { String sql = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, fakeJobUUID, new JobEngineConfig(KylinConfig.getInstanceFromEnv())); System.out.println(sql); - assertEquals(1168, sql.length()); + assertEquals(1132, sql.length()); } }