From 2cbcb190ae2ac632ecd28ed714ececa1d581ffe8 Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Fri, 28 Nov 2014 19:18:43 +0800 Subject: [PATCH] Code refactor for metadata structure change; --- .../main/java/com/kylinolap/cube/CubeInstance.java | 2 +- .../main/java/com/kylinolap/cube/CubeManager.java | 13 +-- .../com/kylinolap/cube/CubeSegmentValidator.java | 7 +- .../java/com/kylinolap/cube/cuboid/CuboidCLI.java | 4 +- .../java/com/kylinolap/cube/model/CubeDesc.java | 79 ++++++++++------- .../com/kylinolap/cube/project/ProjectManager.java | 4 +- .../cube/AggregationGroupSizeRuleTest.java | 99 ++++++++++++++++++++++ .../test/java/com/kylinolap/cube/CubeDescTest.java | 59 +++++++++++++ .../com/kylinolap/cube/CubeManagerCacheTest.java | 6 +- .../java/com/kylinolap/cube/CubeManagerTest.java | 6 +- .../com/kylinolap/cube/DictionaryManagerTest.java | 82 ++++++++++++++++++ .../kylinolap/cube/MandatoryColumnRuleTest.java | 56 ++++++++++++ .../com/kylinolap/cube/RowKeyAttrRuleTest.java | 57 +++++++++++++ .../com/kylinolap/cube/SegmentManagementTest.java | 18 ++-- .../kylinolap/cube/cuboid/CuboidSchedulerTest.java | 19 +++-- .../java/com/kylinolap/cube/cuboid/CuboidTest.java | 12 ++- .../kylinolap/cube/project/ProjectManagerTest.java | 8 +- .../java/com/kylinolap/dict/DictionaryManager.java | 30 +++---- .../com/kylinolap/dict/DictionaryManagerTest.java | 80 ----------------- .../job/hadoop/cube/FactDistinctColumnsMapper.java | 4 +- .../job/hadoop/cube/MergeCuboidMapper.java | 3 +- .../kylinolap/metadata/model/DataModelDesc.java | 53 ++++++++++++ .../com/kylinolap/metadata/model/LookupDesc.java | 17 ++++ .../java/com/kylinolap/metadata/CubeDescTest.java | 50 ----------- .../kylinolap/metadata/MetadataManagerTest.java | 7 -- .../rule/AggregationGroupSizeRuleTest.java | 99 ---------------------- .../validation/rule/MandatoryColumnRuleTest.java | 56 ------------ .../validation/rule/RowKeyAttrRuleTest.java | 57 ------------- .../com/kylinolap/rest/service/BasicService.java | 2 +- .../com/kylinolap/rest/service/CubeService.java | 10 +-- .../rest/controller/CubeControllerTest.java | 2 +- 31 files changed, 558 insertions(+), 443 deletions(-) create mode 100644 cube/src/test/java/com/kylinolap/cube/AggregationGroupSizeRuleTest.java create mode 100644 cube/src/test/java/com/kylinolap/cube/CubeDescTest.java create mode 100644 cube/src/test/java/com/kylinolap/cube/DictionaryManagerTest.java create mode 100644 cube/src/test/java/com/kylinolap/cube/MandatoryColumnRuleTest.java create mode 100644 cube/src/test/java/com/kylinolap/cube/RowKeyAttrRuleTest.java delete mode 100644 dictionary/src/test/java/com/kylinolap/dict/DictionaryManagerTest.java delete mode 100644 metadata/src/test/java/com/kylinolap/metadata/CubeDescTest.java delete mode 100644 metadata/src/test/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRuleTest.java delete mode 100644 metadata/src/test/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRuleTest.java delete mode 100644 metadata/src/test/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRuleTest.java diff --git a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java index 2e33e5d..fd7abb0 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java @@ -166,7 +166,7 @@ public long getAllocatedStartDate() { } public CubeDesc getDescriptor() { - return MetadataManager.getInstance(config).getCubeDesc(descName); + return CubeManager.getInstance(config).getCubeDesc(descName); } public InvertedIndexDesc getInvertedIndexDesc() { diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java index 2341425..fd546d1 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java @@ -335,7 +335,7 @@ public void buildInvertedIndexDictionary(CubeSegment cubeSeg, String factColumns if (iiDesc.isMetricsCol(col)) continue; - DictionaryInfo dict = dictMgr.buildDictionary(null, col, factColumnsPath); + DictionaryInfo dict = dictMgr.buildDictionary(null, null, col, factColumnsPath); cubeSeg.putDictResPath(col, dict.getResourcePath()); } @@ -343,11 +343,12 @@ public void buildInvertedIndexDictionary(CubeSegment cubeSeg, String factColumns } public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, String factColumnsPath) throws IOException { - if (!cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) + CubeDesc cubeDesc = cubeSeg.getCubeDesc(); + if (!cubeDesc.getRowkey().isUseDictionary(col)) return null; DictionaryManager dictMgr = getDictionaryManager(); - DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeSeg.getCubeDesc(), col, factColumnsPath); + DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath); cubeSeg.putDictResPath(col, dictInfo.getResourcePath()); saveResource(cubeSeg.getCubeInstance()); @@ -639,10 +640,12 @@ private void makeDictForNewSegment(CubeInstance cube, CubeSegment newSeg, List colsNeedCopyDict = new HashSet(); DictionaryManager dictMgr = this.getDictionaryManager(); - for (DimensionDesc dim : cube.getDescriptor().getDimensions()) { + CubeDesc cubeDesc = cube.getDescriptor(); + for (DimensionDesc dim : cubeDesc.getDimensions()) { for (TblColRef col : dim.getColumnRefs()) { if (newSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { - if (cube.getDescriptor().getFactTable().equalsIgnoreCase((String) dictMgr.decideSourceData(cube.getDescriptor(), col, null)[0])) { + String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; + if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) { colsNeedMeringDict.add(col); } else { colsNeedCopyDict.add(col); diff --git a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java index 0cc1939..3f50032 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java @@ -22,6 +22,7 @@ import java.util.List; import com.kylinolap.cube.exception.CubeIntegrityException; +import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.cube.model.DimensionDesc; import com.kylinolap.cube.model.CubePartitionDesc.CubePartitionType; import com.kylinolap.dict.DictionaryManager; @@ -87,11 +88,13 @@ private void checkLoopTableConsistency(CubeInstance cube, List newS List segmentList = cube.getMergingSegments(cubeSeg); HashSet cols = new HashSet(); - for (DimensionDesc dim : cube.getDescriptor().getDimensions()) { + CubeDesc cubeDesc = cube.getDescriptor(); + for (DimensionDesc dim : cubeDesc.getDimensions()) { for (TblColRef col : dim.getColumnRefs()) { // include those dictionaries that do not need mergning try { - if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col) && !cube.getDescriptor().getFactTable().equalsIgnoreCase((String) dictMgr.decideSourceData(cube.getDescriptor(), col, null)[0])) { + String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; + if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col) && !cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) { cols.add(col); } } catch (IOException e) { diff --git a/cube/src/main/java/com/kylinolap/cube/cuboid/CuboidCLI.java b/cube/src/main/java/com/kylinolap/cube/cuboid/CuboidCLI.java index dd4605e..380a073 100644 --- a/cube/src/main/java/com/kylinolap/cube/cuboid/CuboidCLI.java +++ b/cube/src/main/java/com/kylinolap/cube/cuboid/CuboidCLI.java @@ -22,11 +22,11 @@ import java.util.TreeSet; import com.kylinolap.common.KylinConfig; +import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.cube.model.RowKeyDesc; import com.kylinolap.cube.model.RowKeyDesc.AggrGroupMask; import com.kylinolap.cube.model.RowKeyDesc.HierarchyMask; -import com.kylinolap.metadata.MetadataManager; /** * @author yangli9 @@ -35,7 +35,7 @@ public class CuboidCLI { public static void main(String[] args) throws IOException { - MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + CubeManager metaMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); if ("test".equals(args[0])) { CubeDesc cubeDesc = metaMgr.getCubeDesc(args[1]); diff --git a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java index bbaf36d..51130f9 100644 --- a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java +++ b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java @@ -91,11 +91,12 @@ public String toString() { } private KylinConfig config; - + private DataModelDesc model; + @JsonProperty("name") private String name; - @JsonProperty("data_model") - private DataModelDesc model; + @JsonProperty("model_name") + private String modelName; @JsonProperty("description") private String description; @JsonProperty("fact_table") @@ -204,27 +205,27 @@ public DimensionDesc findDimensionByName(String dimName) { return null; } - public TblColRef findPKByFK(TblColRef fk) { - assert isFactTable(fk.getTable()); - - TblColRef candidate = null; - - for (DimensionDesc dim : dimensions) { - JoinDesc join = dim.getJoin(); - if (join == null) - continue; - - int find = ArrayUtils.indexOf(join.getForeignKeyColumns(), fk); - if (find >= 0) { - candidate = join.getPrimaryKeyColumns()[find]; - if (join.getForeignKeyColumns().length == 1) { // is single - // column join? - break; - } - } - } - return candidate; - } +// public TblColRef findPKByFK(TblColRef fk) { +// assert isFactTable(fk.getTable()); +// +// TblColRef candidate = null; +// +// for (DimensionDesc dim : dimensions) { +// JoinDesc join = dim.getJoin(); +// if (join == null) +// continue; +// +// int find = ArrayUtils.indexOf(join.getForeignKeyColumns(), fk); +// if (find >= 0) { +// candidate = join.getPrimaryKeyColumns()[find]; +// if (join.getForeignKeyColumns().length == 1) { // is single +// // column join? +// break; +// } +// } +// } +// return candidate; +// } /** * Get all functions from each measure. @@ -258,9 +259,9 @@ public TblColRef findPKByFK(TblColRef fk) { return result; } - public boolean isFactTable(String factTable) { - return this.factTable.equalsIgnoreCase(factTable); - } +// public boolean isFactTable(String factTable) { +// return this.factTable.equalsIgnoreCase(factTable); +// } public boolean isDerived(TblColRef col) { return derivedToHostMap.containsKey(col); @@ -327,6 +328,22 @@ public void setName(String name) { this.name = name; } + public String getModelName() { + return modelName; + } + + public void setModelName(String modelName) { + this.modelName = modelName; + } + + public DataModelDesc getModel() { + return model; + } + + public void setModel(DataModelDesc model) { + this.model = model; + } + public String getDescription() { return description; } @@ -336,11 +353,7 @@ public void setDescription(String description) { } public String getFactTable() { - return factTable; - } - - public void setFactTable(String factTable) { - this.factTable = factTable; + return model.getFactTable(); } public String[] getNullStrings() { @@ -481,6 +494,8 @@ public void init(KylinConfig config, Map tables) { initJoinColumns(tables); initDimensionColumns(tables); initMeasureColumns(tables); + + //TODO create the DataModel object from MetadataManager and then set here; rowkey.init(this); if (hbaseMapping != null) { diff --git a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java b/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java index c744100..fba564a 100644 --- a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java +++ b/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java @@ -325,7 +325,7 @@ public boolean isExposedColumn(String project, String table, String col) { List cubes = new ArrayList(); ProjectTable projectTable = this.getProjectTable(project, factTableName); for (CubeInstance cube : projectTable.getCubes()) { - if (cube.getDescriptor().isFactTable(factTableName) && cube.isReady()) { + if (cube.getDescriptor().getModel().isFactTable(factTableName) && cube.isReady()) { cubes.add(cube); } } @@ -340,7 +340,7 @@ public boolean isExposedColumn(String project, String table, String col) { for (CubeInstance cube : getProjectTable(project, factTable).getCubes()) { if (cube.isReady() == false) continue; - if (cube.getDescriptor().isFactTable(factTable) == false) + if (cube.getDescriptor().getModel().isFactTable(factTable) == false) continue; relatedDesc.add(cube.getDescriptor()); diff --git a/cube/src/test/java/com/kylinolap/cube/AggregationGroupSizeRuleTest.java b/cube/src/test/java/com/kylinolap/cube/AggregationGroupSizeRuleTest.java new file mode 100644 index 0000000..a76a16a --- /dev/null +++ b/cube/src/test/java/com/kylinolap/cube/AggregationGroupSizeRuleTest.java @@ -0,0 +1,99 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.cube; + +import static org.junit.Assert.*; + +import org.junit.Before; +import org.junit.Test; + +import com.kylinolap.common.util.JsonUtil; +import com.kylinolap.cube.model.CubeDesc; +import com.kylinolap.cube.model.validation.IValidatorRule; +import com.kylinolap.cube.model.validation.ValidateContext; +import com.kylinolap.cube.model.validation.rule.AggregationGroupSizeRule; + +/** + * @author jianliu + * + */ +public class AggregationGroupSizeRuleTest { + + private CubeDesc cube; + private ValidateContext vContext = new ValidateContext(); + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class); + this.cube = desc2; + + } + + @Test + public void testOneMandatoryColumn() { + IValidatorRule rule = new AggregationGroupSizeRule() { + /* + * (non-Javadoc) + * + * @see + * com.kylinolap.metadata.validation.rule.AggregationGroupSizeRule + * #getMaxAgrGroupSize() + */ + @Override + protected int getMaxAgrGroupSize() { + return 3; + } + }; + rule.validate(cube, vContext); + vContext.print(System.out); + assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2); + assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number")); + assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number")); + // assertTrue("Failed to validate aggragation group error", + // vContext.getResults()[2].getMessage() + // .startsWith("Hierachy column")); + } + + @Test + public void testAggColumnSize() { + AggregationGroupSizeRule rule = new AggregationGroupSizeRule() { + /* + * (non-Javadoc) + * + * @see + * com.kylinolap.metadata.validation.rule.AggregationGroupSizeRule + * #getMaxAgrGroupSize() + */ + @Override + protected int getMaxAgrGroupSize() { + return 20; + } + }; + rule.validate(cube, vContext); + vContext.print(System.out); + assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0); + // assertTrue("Failed to validate aggragation group error", + // vContext.getResults()[0].getMessage() + // .startsWith("Aggregation group")); + // assertTrue("Failed to validate aggragation group error", + // vContext.getResults()[0].getMessage() + // .startsWith("Hierachy column")); + } +} diff --git a/cube/src/test/java/com/kylinolap/cube/CubeDescTest.java b/cube/src/test/java/com/kylinolap/cube/CubeDescTest.java new file mode 100644 index 0000000..45cca7b --- /dev/null +++ b/cube/src/test/java/com/kylinolap/cube/CubeDescTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.cube; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.kylinolap.common.util.JsonUtil; +import com.kylinolap.common.util.LocalFileMetadataTestCase; +import com.kylinolap.cube.model.CubeDesc; + +/** + * @author yangli9 + */ +public class CubeDescTest extends LocalFileMetadataTestCase { + + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + @Test + public void testSerialize() throws Exception { + CubeDesc desc = CubeManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); + String str = JsonUtil.writeValueAsIndentString(desc); + System.out.println(str); + @SuppressWarnings("unused") + CubeDesc desc2 = JsonUtil.readValue(str, CubeDesc.class); + } + + + @Test + public void testGetCubeDesc() throws Exception { + CubeDesc cubeDesc = CubeManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); + Assert.assertNotNull(cubeDesc); + } + +} diff --git a/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java b/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java index ddd9b06..87c0734 100644 --- a/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java +++ b/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java @@ -56,7 +56,7 @@ public void testReloadCache() throws Exception { // clean legacy in case last run failed store.deleteResource("/cube/a_whole_new_cube.json"); - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); cubeManager.createCube("a_whole_new_cube", "default", desc, null); @@ -69,7 +69,7 @@ public void testReloadCache() throws Exception { assertEquals(CubeStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus()); } - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(getTestConfig()); + private CubeManager getCubeManager() { + return CubeManager.getInstance(getTestConfig()); } } diff --git a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java index f0dd717..0e8b0af 100644 --- a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java @@ -67,7 +67,7 @@ public void testCreateAndDrop() throws Exception { // clean legacy in case last run failed store.deleteResource("/cube/a_whole_new_cube.json"); - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("a_whole_new_cube")); @@ -82,7 +82,7 @@ public void testCreateAndDrop() throws Exception { assertNull(CubeManager.getInstance(this.getTestConfig()).getCube("a_whole_new_cube")); } - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(getTestConfig()); + private CubeManager getCubeManager() { + return CubeManager.getInstance(getTestConfig()); } } diff --git a/cube/src/test/java/com/kylinolap/cube/DictionaryManagerTest.java b/cube/src/test/java/com/kylinolap/cube/DictionaryManagerTest.java new file mode 100644 index 0000000..540e453 --- /dev/null +++ b/cube/src/test/java/com/kylinolap/cube/DictionaryManagerTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.kylinolap.cube; + +import static org.junit.Assert.*; + +import java.util.HashSet; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.kylinolap.common.util.JsonUtil; +import com.kylinolap.common.util.LocalFileMetadataTestCase; +import com.kylinolap.cube.model.CubeDesc; +import com.kylinolap.dict.Dictionary; +import com.kylinolap.dict.DictionaryInfo; +import com.kylinolap.dict.DictionaryManager; +import com.kylinolap.metadata.model.realization.TblColRef; + +public class DictionaryManagerTest extends LocalFileMetadataTestCase { + + DictionaryManager dictMgr; + + @Before + public void setup() throws Exception { + createTestMetadata(); + dictMgr = DictionaryManager.getInstance(this.getTestConfig()); + } + + @After + public void after() throws Exception { + cleanupTestMetadata(); + } + + @Test + @Ignore + public void basic() throws Exception { + CubeDesc cubeDesc = CubeManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc"); + TblColRef col = cubeDesc.findColumnRef("TEST_SITES", "SITE_NAME"); + + DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null); + System.out.println(JsonUtil.writeValueAsIndentString(info1)); + + DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null); + System.out.println(JsonUtil.writeValueAsIndentString(info2)); + + assertTrue(info1.getUuid() == info2.getUuid()); + + assertTrue(info1 == dictMgr.getDictionaryInfo(info1.getResourcePath())); + assertTrue(info2 == dictMgr.getDictionaryInfo(info2.getResourcePath())); + + assertTrue(info1.getDictionaryObject() == info2.getDictionaryObject()); + + touchDictValues(info1); + } + + @SuppressWarnings("unchecked") + private void touchDictValues(DictionaryInfo info1) { + Dictionary dict = (Dictionary) info1.getDictionaryObject(); + + HashSet set = new HashSet(); + for (int i = 0, n = info1.getCardinality(); i < n; i++) { + set.add(dict.getValueFromId(i)); + } + assertEquals(info1.getCardinality(), set.size()); + } +} diff --git a/cube/src/test/java/com/kylinolap/cube/MandatoryColumnRuleTest.java b/cube/src/test/java/com/kylinolap/cube/MandatoryColumnRuleTest.java new file mode 100644 index 0000000..6ab0906 --- /dev/null +++ b/cube/src/test/java/com/kylinolap/cube/MandatoryColumnRuleTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.cube; + +import static org.junit.Assert.*; + +import org.junit.Before; +import org.junit.Test; + +import com.kylinolap.common.util.JsonUtil; +import com.kylinolap.cube.model.CubeDesc; +import com.kylinolap.cube.model.validation.IValidatorRule; +import com.kylinolap.cube.model.validation.ValidateContext; +import com.kylinolap.cube.model.validation.rule.MandatoryColumnRule; + +/** + * @author jianliu + * + */ +public class MandatoryColumnRuleTest { + + private CubeDesc cube; + private ValidateContext vContext = new ValidateContext(); + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST1_desc.json"), CubeDesc.class); + this.cube = desc2; + + } + + @Test + public void testOneMandatoryColumn() { + IValidatorRule rule = new MandatoryColumnRule(); + rule.validate(cube, vContext); + assertTrue("Failed to validate mandatory error", vContext.getResults().length == 1); + assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("mandatory column")); + } +} diff --git a/cube/src/test/java/com/kylinolap/cube/RowKeyAttrRuleTest.java b/cube/src/test/java/com/kylinolap/cube/RowKeyAttrRuleTest.java new file mode 100644 index 0000000..a37def4 --- /dev/null +++ b/cube/src/test/java/com/kylinolap/cube/RowKeyAttrRuleTest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.cube; + +import static org.junit.Assert.*; + +import org.junit.Before; +import org.junit.Test; + +import com.kylinolap.common.util.JsonUtil; +import com.kylinolap.cube.model.CubeDesc; +import com.kylinolap.cube.model.validation.IValidatorRule; +import com.kylinolap.cube.model.validation.ValidateContext; +import com.kylinolap.cube.model.validation.rule.RowKeyAttrRule; + +/** + * @author jianliu + * + */ +public class RowKeyAttrRuleTest { + + private CubeDesc cube; + private ValidateContext vContext = new ValidateContext(); + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST3_desc.json"), CubeDesc.class); + this.cube = desc2; + + } + + @Test + public void testOneMandatoryColumn() { + IValidatorRule rule = new RowKeyAttrRule(); + rule.validate(cube, vContext); + vContext.print(System.out); + assertTrue("Failed to validate rowkey", vContext.getResults().length == 1); + assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("Rowkey column")); + } +} diff --git a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java index 1687148..49f8577 100644 --- a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java +++ b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java @@ -73,7 +73,7 @@ private void createNewCube(CubeDesc cubeDesc) throws IOException { @Test public void testInitialAndAppend() throws ParseException, IOException, CubeIntegrityException { // create a new cube - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); createNewCube(desc); @@ -212,7 +212,7 @@ public void testMergeSegments() throws IOException, CubeIntegrityException { @Test public void testNonPartitionedCube() throws ParseException, IOException, CubeIntegrityException { // create a new cube - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_desc"); createNewCube(desc); @@ -283,7 +283,7 @@ public void testNonPartitionedCube() throws ParseException, IOException, CubeInt @Test(expected = CubeIntegrityException.class) public void testInvalidAppend() throws ParseException, IOException, CubeIntegrityException { // create a new cube - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); createNewCube(desc); @@ -332,7 +332,7 @@ public void testInvalidAppend() throws ParseException, IOException, CubeIntegrit @Test public void testInitialAndUpsert() throws ParseException, IOException, CubeIntegrityException { // create a new cube - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); createNewCube(desc); @@ -488,7 +488,7 @@ public void testInitialAndUpsert() throws ParseException, IOException, CubeInteg @Test public void testInitialAndUpsert2() throws ParseException, IOException, CubeIntegrityException { // create a new cube - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); createNewCube(desc); @@ -603,7 +603,7 @@ public void testInitialAndUpsert2() throws ParseException, IOException, CubeInte @Test(expected = CubeIntegrityException.class) public void testInvalidUpsert() throws IOException, CubeIntegrityException, ParseException { // create a new cube - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); createNewCube(desc); @@ -654,7 +654,7 @@ public void testInvalidUpsert() throws IOException, CubeIntegrityException, Pars @Test(expected = CubeIntegrityException.class) public void testInvalidUpsert2() throws IOException, CubeIntegrityException, ParseException { // create a new cube - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); createNewCube(desc); @@ -707,4 +707,8 @@ public void testInvalidUpsert2() throws IOException, CubeIntegrityException, Par private MetadataManager getMetadataManager() { return MetadataManager.getInstance(getTestConfig()); } + + private CubeManager getCubeManager() { + return CubeManager.getInstance(getTestConfig()); + } } diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java index 6304b6e..7b968e1 100644 --- a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java @@ -28,6 +28,7 @@ import org.junit.Test; import com.kylinolap.common.util.LocalFileMetadataTestCase; +import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.metadata.MetadataManager; @@ -68,15 +69,15 @@ static String toString(Collection cuboids) { } private CubeDesc getTestKylinCubeWithoutSeller() { - return getMetadataManager().getCubeDesc("test_kylin_cube_without_slr_desc"); + return getCubeManager().getCubeDesc("test_kylin_cube_without_slr_desc"); } private CubeDesc getTestKylinCubeWithSeller() { - return getMetadataManager().getCubeDesc("test_kylin_cube_with_slr_desc"); + return getCubeManager().getCubeDesc("test_kylin_cube_with_slr_desc"); } private CubeDesc getTestKylinCubeWithoutSellerLeftJoin() { - return getMetadataManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); + return getCubeManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); } @Test @@ -248,28 +249,28 @@ public void testCuboidGeneration3() { @Test @Ignore public void testCuboidGeneration4() { - CubeDesc cube = getMetadataManager().getCubeDesc("geox_trans_mtrc_sd_cube_desc"); + CubeDesc cube = getCubeManager().getCubeDesc("geox_trans_mtrc_sd_cube_desc"); CuboidCLI.simulateCuboidGeneration(cube); } @Test @Ignore public void testCuboidGeneration5() { - CubeDesc cube = getMetadataManager().getCubeDesc("clsfd_ga_dayweek"); + CubeDesc cube = getCubeManager().getCubeDesc("clsfd_ga_dayweek"); CuboidCLI.simulateCuboidGeneration(cube); } @Test @Ignore public void testCuboidGeneration6() { - CubeDesc cube = getMetadataManager().getCubeDesc("clsfd_ga_day"); + CubeDesc cube = getCubeManager().getCubeDesc("clsfd_ga_day"); CuboidCLI.simulateCuboidGeneration(cube); } @Test @Ignore public void testCuboidGeneration7() { - CubeDesc cube = getMetadataManager().getCubeDesc("clsfd_ga_week"); + CubeDesc cube = getCubeManager().getCubeDesc("clsfd_ga_week"); CuboidCLI.simulateCuboidGeneration(cube); } @@ -299,6 +300,10 @@ private String sortToString(Collection longs) { private MetadataManager getMetadataManager() { return MetadataManager.getInstance(getTestConfig()); } + + private CubeManager getCubeManager() { + return CubeManager.getInstance(getTestConfig()); + } private void printCount(int[] counts) { int sum = 0; diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java index 2c87f8f..6c0c816 100644 --- a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java +++ b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java @@ -22,6 +22,7 @@ import org.junit.Test; import com.kylinolap.common.util.LocalFileMetadataTestCase; +import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.metadata.MetadataManager; @@ -38,16 +39,21 @@ private MetadataManager getMetadataManager() { return MetadataManager.getInstance(getTestConfig()); } + private CubeManager getCubeManager() { + return CubeManager.getInstance(getTestConfig()); + } + + private CubeDesc getTestKylinCubeII() { - return getMetadataManager().getCubeDesc("test_kylin_cube_ii"); + return getCubeManager().getCubeDesc("test_kylin_cube_ii"); } private CubeDesc getTestKylinCubeWithoutSeller() { - return getMetadataManager().getCubeDesc("test_kylin_cube_without_slr_desc"); + return getCubeManager().getCubeDesc("test_kylin_cube_without_slr_desc"); } private CubeDesc getTestKylinCubeWithSeller() { - return getMetadataManager().getCubeDesc("test_kylin_cube_with_slr_desc"); + return getCubeManager().getCubeDesc("test_kylin_cube_with_slr_desc"); } @Before diff --git a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java b/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java index 850bf9b..0d14862 100644 --- a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java @@ -71,7 +71,7 @@ public void testNewProject() throws Exception { // clean legacy in case last run failed store.deleteResource("/cube/cube_in_alien_project.json"); - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("cube_in_alien_project", "alien", desc, null); assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("cube_in_alien_project")); @@ -115,7 +115,7 @@ public void testExistingProject() throws Exception { // clean legacy in case last run failed store.deleteResource("/cube/new_cube_in_default.json"); - MetadataManager metaMgr = getMetadataManager(); + CubeManager metaMgr = getCubeManager(); CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("new_cube_in_default")); @@ -162,4 +162,8 @@ public void testProjectsLoadAfterProjectChange() throws IOException { private MetadataManager getMetadataManager() { return MetadataManager.getInstance(getTestConfig()); } + + private CubeManager getCubeManager() { + return CubeManager.getInstance(getTestConfig()); + } } diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java index 985848c..668b0ef 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java +++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryManager.java @@ -33,12 +33,12 @@ import com.kylinolap.common.KylinConfig; import com.kylinolap.common.persistence.ResourceStore; import com.kylinolap.common.util.HadoopUtil; -import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.dict.lookup.FileTable; import com.kylinolap.dict.lookup.HiveTable; import com.kylinolap.dict.lookup.ReadableTable; import com.kylinolap.dict.lookup.TableSignature; import com.kylinolap.metadata.MetadataManager; +import com.kylinolap.metadata.model.DataModelDesc; import com.kylinolap.metadata.model.realization.TblColRef; public class DictionaryManager { @@ -147,9 +147,9 @@ public DictionaryInfo mergeDictionary(List dicts) throws IOExcep return trySaveNewDict(newDict, newDictInfo); } - public DictionaryInfo buildDictionary(CubeDesc cube, TblColRef col, String factColumnsPath) throws IOException { + public DictionaryInfo buildDictionary(DataModelDesc model, String dict, TblColRef col, String factColumnsPath) throws IOException { - Object[] tmp = decideSourceData(cube, col, factColumnsPath); + Object[] tmp = decideSourceData(model, dict, col, factColumnsPath); String srcTable = (String) tmp[0]; String srcCol = (String) tmp[1]; int srcColIdx = (Integer) tmp[2]; @@ -163,9 +163,9 @@ public DictionaryInfo buildDictionary(CubeDesc cube, TblColRef col, String factC return getDictionaryInfo(dupDict); } - Dictionary dict = DictionaryGenerator.buildDictionary(dictInfo, inpTable); + Dictionary dictionary = DictionaryGenerator.buildDictionary(dictInfo, inpTable); - return trySaveNewDict(dict, dictInfo); + return trySaveNewDict(dictionary, dictInfo); } /** @@ -176,7 +176,7 @@ public DictionaryInfo buildDictionary(CubeDesc cube, TblColRef col, String factC * 3. column cardinal in source table * 4. ReadableTable object */ - public Object[] decideSourceData(CubeDesc cube, TblColRef col, String factColumnsPath) throws IOException { + public Object[] decideSourceData(DataModelDesc model, String dict, TblColRef col, String factColumnsPath) throws IOException { String srcTable; String srcCol; int srcColIdx; @@ -184,7 +184,7 @@ public DictionaryInfo buildDictionary(CubeDesc cube, TblColRef col, String factC MetadataManager metaMgr = MetadataManager.getInstance(config); // case of full table (dict on fact table) - if (cube == null) { + if (model == null) { srcTable = col.getTable(); srcCol = col.getName(); srcColIdx = col.getColumn().getZeroBasedIndex(); @@ -200,13 +200,13 @@ public DictionaryInfo buildDictionary(CubeDesc cube, TblColRef col, String factC // Note FK on fact table is supported by scan the related PK on lookup // table - String useDict = cube.getRowkey().getDictionary(col); + //String useDict = cube.getRowkey().getDictionary(col); // normal case, source from lookup table - if ("true".equals(useDict) || "string".equals(useDict) || "number".equals(useDict) || "any".equals(useDict)) { + if ("true".equals(dict) || "string".equals(dict) || "number".equals(dict) || "any".equals(dict)) { // FK on fact table, use PK from lookup instead - if (cube.isFactTable(col.getTable())) { - TblColRef pkCol = cube.findPKByFK(col); + if (model.isFactTable(col.getTable())) { + TblColRef pkCol = model.findPKByFK(col); if (pkCol != null) col = pkCol; // scan the counterparty PK on lookup table // instead @@ -214,7 +214,7 @@ public DictionaryInfo buildDictionary(CubeDesc cube, TblColRef col, String factC srcTable = col.getTable(); srcCol = col.getName(); srcColIdx = col.getColumn().getZeroBasedIndex(); - if (cube.isFactTable(col.getTable())) { + if (model.isFactTable(col.getTable())) { table = new FileTable(factColumnsPath + "/" + col.getName(), -1); } else { table = new HiveTable(metaMgr, col.getTable()); @@ -223,11 +223,11 @@ public DictionaryInfo buildDictionary(CubeDesc cube, TblColRef col, String factC // otherwise could refer to a data set, e.g. common_indicators.txt // (LEGACY PATH, since distinct values are collected from fact table) else { - String dictDataSetPath = unpackDataSet(this.config.getTempHDFSDir(), useDict); + String dictDataSetPath = unpackDataSet(this.config.getTempHDFSDir(), dict); if (dictDataSetPath == null) - throw new IllegalArgumentException("Unknown dictionary data set '" + useDict + "', referred from " + col); + throw new IllegalArgumentException("Unknown dictionary data set '" + dict + "', referred from " + col); srcTable = "PREDEFINED"; - srcCol = useDict; + srcCol = dict; srcColIdx = 0; table = new FileTable(dictDataSetPath, -1); } diff --git a/dictionary/src/test/java/com/kylinolap/dict/DictionaryManagerTest.java b/dictionary/src/test/java/com/kylinolap/dict/DictionaryManagerTest.java deleted file mode 100644 index 790e0cf..0000000 --- a/dictionary/src/test/java/com/kylinolap/dict/DictionaryManagerTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kylinolap.dict; - -import static org.junit.Assert.*; - -import java.util.HashSet; - -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import com.kylinolap.common.util.JsonUtil; -import com.kylinolap.common.util.LocalFileMetadataTestCase; -import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.metadata.MetadataManager; -import com.kylinolap.metadata.model.realization.TblColRef; - -public class DictionaryManagerTest extends LocalFileMetadataTestCase { - - DictionaryManager dictMgr; - - @Before - public void setup() throws Exception { - createTestMetadata(); - dictMgr = DictionaryManager.getInstance(this.getTestConfig()); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - @Test - @Ignore - public void basic() throws Exception { - CubeDesc cubeDesc = MetadataManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc"); - TblColRef col = cubeDesc.findColumnRef("TEST_SITES", "SITE_NAME"); - - DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc, col, null); - System.out.println(JsonUtil.writeValueAsIndentString(info1)); - - DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc, col, null); - System.out.println(JsonUtil.writeValueAsIndentString(info2)); - - assertTrue(info1.getUuid() == info2.getUuid()); - - assertTrue(info1 == dictMgr.getDictionaryInfo(info1.getResourcePath())); - assertTrue(info2 == dictMgr.getDictionaryInfo(info2.getResourcePath())); - - assertTrue(info1.getDictionaryObject() == info2.getDictionaryObject()); - - touchDictValues(info1); - } - - @SuppressWarnings("unchecked") - private void touchDictValues(DictionaryInfo info1) { - Dictionary dict = (Dictionary) info1.getDictionaryObject(); - - HashSet set = new HashSet(); - for (int i = 0, n = info1.getCardinality(); i < n; i++) { - set.add(dict.getValueFromId(i)); - } - assertEquals(info1.getCardinality(), set.size()); - } -} diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsMapper.java index bd6d1cc..9a98b23 100644 --- a/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsMapper.java +++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/FactDistinctColumnsMapper.java @@ -81,8 +81,8 @@ protected void setup(Context context) throws IOException { if (rowkey.isUseDictionary(col) == false) continue; - String scanTable = (String) dictMgr.decideSourceData(cubeDesc, col, null)[0]; - if (cubeDesc.isFactTable(scanTable)) { + String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; + if (cubeDesc.getModel().isFactTable(scanTable)) { System.out.println(col + " -- " + i); factDictCols.add(i); } diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java index f32a5f2..fd051c1 100644 --- a/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java +++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java @@ -72,7 +72,8 @@ private Boolean checkNeedMerging(TblColRef col) throws IOException { if (ret != null) return ret; else { - ret = cubeDesc.getRowkey().isUseDictionary(col) && cubeDesc.getFactTable().equalsIgnoreCase((String) DictionaryManager.getInstance(config).decideSourceData(cubeDesc, col, null)[0]); + String dictTable = (String) DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; + ret = cubeDesc.getRowkey().isUseDictionary(col) && cubeDesc.getFactTable().equalsIgnoreCase(dictTable); dictsNeedMerging.put(col, ret); return ret; } diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java index 6489143..3e95ee9 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java +++ b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java @@ -1,8 +1,11 @@ package com.kylinolap.metadata.model; +import org.apache.commons.lang.ArrayUtils; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import com.kylinolap.metadata.model.realization.TblColRef; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class DataModelDesc { @@ -15,5 +18,55 @@ @JsonProperty("lookups") private LookupDesc[] lookups; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFactTable() { + return factTable; + } + + public void setFactTable(String factTable) { + this.factTable = factTable; + } + + public LookupDesc[] getLookups() { + return lookups; + } + + public void setLookups(LookupDesc[] lookups) { + this.lookups = lookups; + } + + public boolean isFactTable(String factTable) { + return this.factTable.equalsIgnoreCase(factTable); + } + + public TblColRef findPKByFK(TblColRef fk) { + assert isFactTable(fk.getTable()); + + TblColRef candidate = null; + + for (LookupDesc dim : lookups) { + JoinDesc join = dim.getJoin(); + if (join == null) + continue; + + int find = ArrayUtils.indexOf(join.getForeignKeyColumns(), fk); + if (find >= 0) { + candidate = join.getPrimaryKeyColumns()[find]; + if (join.getForeignKeyColumns().length == 1) { // is single + // column join? + break; + } + } + } + return candidate; + } } diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/LookupDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/LookupDesc.java index 089ded8..fe3cff5 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/model/LookupDesc.java +++ b/metadata/src/main/java/com/kylinolap/metadata/model/LookupDesc.java @@ -12,5 +12,22 @@ @JsonProperty("join") private JoinDesc join; + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public JoinDesc getJoin() { + return join; + } + + public void setJoin(JoinDesc join) { + this.join = join; + } + } diff --git a/metadata/src/test/java/com/kylinolap/metadata/CubeDescTest.java b/metadata/src/test/java/com/kylinolap/metadata/CubeDescTest.java deleted file mode 100644 index 3b723e3..0000000 --- a/metadata/src/test/java/com/kylinolap/metadata/CubeDescTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kylinolap.metadata; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.kylinolap.common.util.JsonUtil; -import com.kylinolap.common.util.LocalFileMetadataTestCase; -import com.kylinolap.cube.model.CubeDesc; - -/** - * @author yangli9 - */ -public class CubeDescTest extends LocalFileMetadataTestCase { - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testSerialize() throws Exception { - CubeDesc desc = MetadataManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); - String str = JsonUtil.writeValueAsIndentString(desc); - System.out.println(str); - @SuppressWarnings("unused") - CubeDesc desc2 = JsonUtil.readValue(str, CubeDesc.class); - } -} diff --git a/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java b/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java index 6fc9797..149a618 100644 --- a/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java +++ b/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java @@ -23,7 +23,6 @@ import org.junit.Test; import com.kylinolap.common.util.LocalFileMetadataTestCase; -import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.metadata.model.TableDesc; /** @@ -64,10 +63,4 @@ public void testGetInstance() throws Exception { Assert.assertTrue(MetadataManager.getInstance(this.getTestConfig()).listAllTables().size() > 0); } - @Test - public void testGetCubeDesc() throws Exception { - CubeDesc cubeDesc = MetadataManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); - Assert.assertNotNull(cubeDesc); - } - } diff --git a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRuleTest.java b/metadata/src/test/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRuleTest.java deleted file mode 100644 index e3e3e7d..0000000 --- a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRuleTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kylinolap.metadata.validation.rule; - -import static org.junit.Assert.*; - -import org.junit.Before; -import org.junit.Test; - -import com.kylinolap.common.util.JsonUtil; -import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.model.validation.IValidatorRule; -import com.kylinolap.cube.model.validation.ValidateContext; -import com.kylinolap.cube.model.validation.rule.AggregationGroupSizeRule; - -/** - * @author jianliu - * - */ -public class AggregationGroupSizeRuleTest { - - private CubeDesc cube; - private ValidateContext vContext = new ValidateContext(); - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class); - this.cube = desc2; - - } - - @Test - public void testOneMandatoryColumn() { - IValidatorRule rule = new AggregationGroupSizeRule() { - /* - * (non-Javadoc) - * - * @see - * com.kylinolap.metadata.validation.rule.AggregationGroupSizeRule - * #getMaxAgrGroupSize() - */ - @Override - protected int getMaxAgrGroupSize() { - return 3; - } - }; - rule.validate(cube, vContext); - vContext.print(System.out); - assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2); - assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number")); - assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number")); - // assertTrue("Failed to validate aggragation group error", - // vContext.getResults()[2].getMessage() - // .startsWith("Hierachy column")); - } - - @Test - public void testAggColumnSize() { - AggregationGroupSizeRule rule = new AggregationGroupSizeRule() { - /* - * (non-Javadoc) - * - * @see - * com.kylinolap.metadata.validation.rule.AggregationGroupSizeRule - * #getMaxAgrGroupSize() - */ - @Override - protected int getMaxAgrGroupSize() { - return 20; - } - }; - rule.validate(cube, vContext); - vContext.print(System.out); - assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0); - // assertTrue("Failed to validate aggragation group error", - // vContext.getResults()[0].getMessage() - // .startsWith("Aggregation group")); - // assertTrue("Failed to validate aggragation group error", - // vContext.getResults()[0].getMessage() - // .startsWith("Hierachy column")); - } -} diff --git a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRuleTest.java b/metadata/src/test/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRuleTest.java deleted file mode 100644 index 4ebe5ab..0000000 --- a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRuleTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kylinolap.metadata.validation.rule; - -import static org.junit.Assert.*; - -import org.junit.Before; -import org.junit.Test; - -import com.kylinolap.common.util.JsonUtil; -import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.model.validation.IValidatorRule; -import com.kylinolap.cube.model.validation.ValidateContext; -import com.kylinolap.cube.model.validation.rule.MandatoryColumnRule; - -/** - * @author jianliu - * - */ -public class MandatoryColumnRuleTest { - - private CubeDesc cube; - private ValidateContext vContext = new ValidateContext(); - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST1_desc.json"), CubeDesc.class); - this.cube = desc2; - - } - - @Test - public void testOneMandatoryColumn() { - IValidatorRule rule = new MandatoryColumnRule(); - rule.validate(cube, vContext); - assertTrue("Failed to validate mandatory error", vContext.getResults().length == 1); - assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("mandatory column")); - } -} diff --git a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRuleTest.java b/metadata/src/test/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRuleTest.java deleted file mode 100644 index 81f1d40..0000000 --- a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRuleTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kylinolap.metadata.validation.rule; - -import static org.junit.Assert.*; - -import org.junit.Before; -import org.junit.Test; - -import com.kylinolap.common.util.JsonUtil; -import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.model.validation.IValidatorRule; -import com.kylinolap.cube.model.validation.ValidateContext; -import com.kylinolap.cube.model.validation.rule.RowKeyAttrRule; - -/** - * @author jianliu - * - */ -public class RowKeyAttrRuleTest { - - private CubeDesc cube; - private ValidateContext vContext = new ValidateContext(); - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST3_desc.json"), CubeDesc.class); - this.cube = desc2; - - } - - @Test - public void testOneMandatoryColumn() { - IValidatorRule rule = new RowKeyAttrRule(); - rule.validate(cube, vContext); - vContext.print(System.out); - assertTrue("Failed to validate rowkey", vContext.getResults().length == 1); - assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("Rowkey column")); - } -} diff --git a/server/src/main/java/com/kylinolap/rest/service/BasicService.java b/server/src/main/java/com/kylinolap/rest/service/BasicService.java index 812c6aa..b76da75 100644 --- a/server/src/main/java/com/kylinolap/rest/service/BasicService.java +++ b/server/src/main/java/com/kylinolap/rest/service/BasicService.java @@ -149,7 +149,7 @@ public KylinConfig getKylinConfig() { public MetadataManager getMetadataManager() { return MetadataManager.getInstance(getConfig()); } - + public CubeManager getCubeManager() { return CubeManager.getInstance(getConfig()); } diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java index fc3bbc3..fa3c181 100644 --- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java +++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java @@ -163,10 +163,10 @@ public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeD CubeDesc createdDesc = null; CubeInstance createdCube = null; - createdDesc = getMetadataManager().createCubeDesc(desc); + createdDesc = getCubeManager().createCubeDesc(desc); if (!createdDesc.getError().isEmpty()) { - getMetadataManager().removeCubeDesc(createdDesc); + getCubeManager().removeCubeDesc(createdDesc); throw new InternalErrorException(createdDesc.getError().get(0)); } @@ -174,7 +174,7 @@ public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeD int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc); logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids"); }catch(Exception e){ - getMetadataManager().removeCubeDesc(createdDesc); + getCubeManager().removeCubeDesc(createdDesc); throw new InternalErrorException("Failed to deal with the request.", e); } @@ -201,7 +201,7 @@ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newPr this.releaseAllSegments(cube); } - CubeDesc updatedCubeDesc = getMetadataManager().updateCubeDesc(desc); + CubeDesc updatedCubeDesc = getCubeManager().updateCubeDesc(desc); int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc); logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids"); @@ -552,7 +552,7 @@ private static void writeResource(InputStream source, KylinConfig dstConfig, Str public void updateCubeNotifyList(CubeInstance cube, List notifyList) throws IOException, CubeIntegrityException { CubeDesc desc = cube.getDescriptor(); desc.setNotifyList(notifyList); - getMetadataManager().updateCubeDesc(desc); + getCubeManager().updateCubeDesc(desc); } public CubeInstance rebuildLookupSnapshot(String cubeName, String segmentName, String lookupTable) throws IOException { diff --git a/server/src/test/java/com/kylinolap/rest/controller/CubeControllerTest.java b/server/src/test/java/com/kylinolap/rest/controller/CubeControllerTest.java index f873683..3f68729 100644 --- a/server/src/test/java/com/kylinolap/rest/controller/CubeControllerTest.java +++ b/server/src/test/java/com/kylinolap/rest/controller/CubeControllerTest.java @@ -70,7 +70,7 @@ public void testBasics() throws IOException { newCube.setHBaseMapping(cube.getHBaseMapping()); newCube.setMeasures(cube.getMeasures()); newCube.setConfig(cube.getConfig()); - newCube.setFactTable(cube.getFactTable()); + newCube.getModel().setFactTable(cube.getFactTable()); newCube.setRowkey(cube.getRowkey()); ObjectMapper mapper = new ObjectMapper();