From 3c2cac15b000845eb62a5fd01b102ea3570a1521 Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Mon, 1 Dec 2014 14:15:11 +0800 Subject: [PATCH 1/3] Extract CubeDesc related code from CubeManager.java to CubeDescManager.java --- .../java/com/kylinolap/cube/CubeDescManager.java | 278 +++++++++++++++++++++ .../main/java/com/kylinolap/cube/CubeInstance.java | 2 +- .../main/java/com/kylinolap/cube/CubeManager.java | 173 ------------- .../java/com/kylinolap/cube/cuboid/CuboidCLI.java | 6 +- .../test/java/com/kylinolap/cube/CubeDescTest.java | 4 +- .../com/kylinolap/cube/CubeManagerCacheTest.java | 8 +- .../java/com/kylinolap/cube/CubeManagerTest.java | 8 +- .../com/kylinolap/cube/DictionaryManagerTest.java | 2 +- .../com/kylinolap/cube/SegmentManagementTest.java | 36 ++- .../kylinolap/cube/cuboid/CuboidSchedulerTest.java | 24 +- .../java/com/kylinolap/cube/cuboid/CuboidTest.java | 17 +- .../kylinolap/cube/project/ProjectManagerTest.java | 13 +- .../com/kylinolap/rest/service/BasicService.java | 5 + .../com/kylinolap/rest/service/CubeService.java | 10 +- 14 files changed, 344 insertions(+), 242 deletions(-) create mode 100644 cube/src/main/java/com/kylinolap/cube/CubeDescManager.java diff --git a/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java b/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java new file mode 100644 index 0000000..033c2d8 --- /dev/null +++ b/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java @@ -0,0 +1,278 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.cube; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.kylinolap.common.KylinConfig; +import com.kylinolap.common.persistence.JsonSerializer; +import com.kylinolap.common.persistence.ResourceStore; +import com.kylinolap.common.persistence.Serializer; +import com.kylinolap.common.restclient.Broadcaster; +import com.kylinolap.common.restclient.SingleValueCache; +import com.kylinolap.cube.model.CubeDesc; +import com.kylinolap.cube.model.validation.CubeMetadataValidator; +import com.kylinolap.cube.model.validation.ValidateContext; +import com.kylinolap.metadata.MetadataConstances; +import com.kylinolap.metadata.MetadataManager; + +/** + * Manager class for CubeDesc; extracted from #CubeManager + * @author shaoshi + * + */ +public class CubeDescManager { + + private static final Logger logger = LoggerFactory.getLogger(CubeDescManager.class); + + private static final Serializer CUBE_DESC_SERIALIZER = new JsonSerializer(CubeDesc.class); + + // static cached instances + private static final ConcurrentHashMap CACHE = new ConcurrentHashMap(); + + // ============================================================================ + + private KylinConfig config; + // name ==> CubeDesc + private SingleValueCache cubeDescMap = new SingleValueCache(Broadcaster.TYPE.METADATA); + + public static CubeDescManager getInstance(KylinConfig config) { + CubeDescManager r = CACHE.get(config); + if (r != null) { + return r; + } + + synchronized (CubeDescManager.class) { + r = CACHE.get(config); + if (r != null) { + return r; + } + try { + r = new CubeDescManager(config); + CACHE.put(config, r); + if (CACHE.size() > 1) { + logger.warn("More than one singleton exist"); + } + return r; + } catch (IOException e) { + throw new IllegalStateException("Failed to init CubeDescManager from " + config, e); + } + } + } + + public static void clearCache() { + CACHE.clear(); + } + + public static synchronized void removeInstance(KylinConfig config) { + CACHE.remove(config); + } + + private CubeDescManager(KylinConfig config) throws IOException { + logger.info("Initializing CubeDescManager with config " + config); + this.config = config; + + } + + public CubeDesc getCubeDesc(String name) { + return cubeDescMap.get(name); + } + + + /** + * Reload CubeDesc from resource store It will be triggered by an desc + * update event. + * + * @param name + * @throws IOException + */ + public CubeDesc reloadCubeDesc(String name) throws IOException { + + // Save Source + String path = CubeDesc.getCubeDescResourcePath(name); + + // Reload the CubeDesc + CubeDesc ndesc = loadCubeDesc(path); + + // Here replace the old one + cubeDescMap.put(ndesc.getName(), ndesc); + return ndesc; + } + + private CubeDesc loadCubeDesc(String path) throws IOException { + ResourceStore store = getStore(); + logger.debug("Loading CubeDesc " + store.getReadableResourcePath(path)); + + CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER); + + if (StringUtils.isBlank(ndesc.getName())) { + throw new IllegalStateException("CubeDesc name must not be blank"); + } + + ndesc.init(config, getMetadataManager().getAllTablesMap()); + + if (ndesc.getError().isEmpty() == false) { + throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError()); + } + + return ndesc; + } + + + /** + * Create a new CubeDesc + * + * @param cubeDesc + * @return + * @throws IOException + */ + public CubeDesc createCubeDesc(CubeDesc cubeDesc) throws IOException { + if (cubeDesc.getUuid() == null || cubeDesc.getName() == null) + throw new IllegalArgumentException(); + if (cubeDescMap.containsKey(cubeDesc.getName())) + throw new IllegalArgumentException("CubeDesc '" + cubeDesc.getName() + "' already exists"); + + try { + cubeDesc.init(config, getMetadataManager().getAllTablesMap()); + } catch (IllegalStateException e) { + cubeDesc.addError(e.getMessage(), true); + } + // Check base validation + if (!cubeDesc.getError().isEmpty()) { + return cubeDesc; + } + // Semantic validation + CubeMetadataValidator validator = new CubeMetadataValidator(); + ValidateContext context = validator.validate(cubeDesc, true); + if (!context.ifPass()) { + return cubeDesc; + } + + cubeDesc.setSignature(cubeDesc.calculateSignature()); + + String path = cubeDesc.getResourcePath(); + getStore().putResource(path, cubeDesc, CUBE_DESC_SERIALIZER); + cubeDescMap.put(cubeDesc.getName(), cubeDesc); + + return cubeDesc; + } + + + + // remove cubeDesc + public void removeCubeDesc(CubeDesc cubeDesc) throws IOException{ + String path = cubeDesc.getResourcePath(); + getStore().deleteResource(path); + cubeDescMap.remove(cubeDesc.getName()); + } + + private void reloadAllCubeDesc() throws IOException { + ResourceStore store = getStore(); + logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT)); + + cubeDescMap.clear(); + + List paths = store.collectResourceRecursively(ResourceStore.CUBE_DESC_RESOURCE_ROOT, MetadataConstances.FILE_SURFIX); + for (String path : paths) { + CubeDesc desc; + try { + desc = loadCubeDesc(path); + } catch (Exception e) { + logger.error("Error loading cube desc " + path, e); + continue; + } + if (path.equals(desc.getResourcePath()) == false) { + logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath()); + continue; + } + if (cubeDescMap.containsKey(desc.getName())) { + logger.error("Dup CubeDesc name '" + desc.getName() + "' on path " + path); + continue; + } + + cubeDescMap.putLocal(desc.getName(), desc); + } + + logger.debug("Loaded " + cubeDescMap.size() + " Cube(s)"); + } + + + /** + * Update CubeDesc with the input. Broadcast the event into cluster + * + * @param desc + * @return + * @throws IOException + */ + public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException { + // Validate CubeDesc + if (desc.getUuid() == null || desc.getName() == null) { + throw new IllegalArgumentException(); + } + String name = desc.getName(); + if (!cubeDescMap.containsKey(name)) { + throw new IllegalArgumentException("CubeDesc '" + name + "' does not exist."); + } + + try { + desc.init(config, getMetadataManager().getAllTablesMap()); + } catch (IllegalStateException e) { + desc.addError(e.getMessage(), true); + return desc; + } catch (IllegalArgumentException e) { + desc.addError(e.getMessage(), true); + return desc; + } + + // Semantic validation + CubeMetadataValidator validator = new CubeMetadataValidator(); + ValidateContext context = validator.validate(desc, true); + if (!context.ifPass()) { + return desc; + } + + desc.setSignature(desc.calculateSignature()); + + // Save Source + String path = desc.getResourcePath(); + getStore().putResource(path, desc, CUBE_DESC_SERIALIZER); + + // Reload the CubeDesc + CubeDesc ndesc = loadCubeDesc(path); + // Here replace the old one + cubeDescMap.put(ndesc.getName(), desc); + + return ndesc; + } + + + + private MetadataManager getMetadataManager() { + return MetadataManager.getInstance(config); + } + + private ResourceStore getStore() { + return ResourceStore.getStore(this.config); + } + +} diff --git a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java index fd7abb0..796eaba 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java @@ -166,7 +166,7 @@ public long getAllocatedStartDate() { } public CubeDesc getDescriptor() { - return CubeManager.getInstance(config).getCubeDesc(descName); + return CubeDescManager.getInstance(config).getCubeDesc(descName); } public InvertedIndexDesc getInvertedIndexDesc() { diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java index fd546d1..1813e7b 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java @@ -40,8 +40,6 @@ import com.kylinolap.cube.exception.CubeIntegrityException; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.cube.model.DimensionDesc; -import com.kylinolap.cube.model.validation.CubeMetadataValidator; -import com.kylinolap.cube.model.validation.ValidateContext; import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.dict.DateStrDictionary; @@ -52,7 +50,6 @@ import com.kylinolap.dict.lookup.LookupStringTable; import com.kylinolap.dict.lookup.SnapshotManager; import com.kylinolap.dict.lookup.SnapshotTable; -import com.kylinolap.metadata.MetadataConstances; import com.kylinolap.metadata.MetadataManager; import com.kylinolap.metadata.model.ColumnDesc; import com.kylinolap.metadata.model.TableDesc; @@ -67,8 +64,6 @@ private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; private static int HBASE_TABLE_LENGTH = 10; - - private static final Serializer CUBE_DESC_SERIALIZER = new JsonSerializer(CubeDesc.class); private static final Serializer CUBE_SERIALIZER = new JsonSerializer(CubeInstance.class); private static final Logger logger = LoggerFactory.getLogger(CubeManager.class); @@ -111,8 +106,6 @@ public static synchronized void removeInstance(KylinConfig config) { // ============================================================================ private KylinConfig config; - // name ==> CubeDesc - private SingleValueCache cubeDescMap = new SingleValueCache(Broadcaster.TYPE.METADATA); // cube name ==> CubeInstance private SingleValueCache cubeMap = new SingleValueCache(Broadcaster.TYPE.CUBE); // "table/column" ==> lookup table @@ -128,172 +121,6 @@ private CubeManager(KylinConfig config) throws IOException { loadAllCubeInstance(); } - public CubeDesc getCubeDesc(String name) { - return cubeDescMap.get(name); - } - - /** - * Create a new CubeDesc - * - * @param cubeDesc - * @return - * @throws IOException - */ - public CubeDesc createCubeDesc(CubeDesc cubeDesc) throws IOException { - if (cubeDesc.getUuid() == null || cubeDesc.getName() == null) - throw new IllegalArgumentException(); - if (cubeDescMap.containsKey(cubeDesc.getName())) - throw new IllegalArgumentException("CubeDesc '" + cubeDesc.getName() + "' already exists"); - - try { - cubeDesc.init(config, getMetadataManager().getAllTablesMap()); - } catch (IllegalStateException e) { - cubeDesc.addError(e.getMessage(), true); - } - // Check base validation - if (!cubeDesc.getError().isEmpty()) { - return cubeDesc; - } - // Semantic validation - CubeMetadataValidator validator = new CubeMetadataValidator(); - ValidateContext context = validator.validate(cubeDesc, true); - if (!context.ifPass()) { - return cubeDesc; - } - - cubeDesc.setSignature(cubeDesc.calculateSignature()); - - String path = cubeDesc.getResourcePath(); - getStore().putResource(path, cubeDesc, CUBE_DESC_SERIALIZER); - cubeDescMap.put(cubeDesc.getName(), cubeDesc); - - return cubeDesc; - } - - // remove cubeDesc - public void removeCubeDesc(CubeDesc cubeDesc) throws IOException{ - String path = cubeDesc.getResourcePath(); - getStore().deleteResource(path); - cubeDescMap.remove(cubeDesc.getName()); - } - - private void reloadAllCubeDesc() throws IOException { - ResourceStore store = getStore(); - logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT)); - - cubeDescMap.clear(); - - List paths = store.collectResourceRecursively(ResourceStore.CUBE_DESC_RESOURCE_ROOT, MetadataConstances.FILE_SURFIX); - for (String path : paths) { - CubeDesc desc; - try { - desc = loadCubeDesc(path); - } catch (Exception e) { - logger.error("Error loading cube desc " + path, e); - continue; - } - if (path.equals(desc.getResourcePath()) == false) { - logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath()); - continue; - } - if (cubeDescMap.containsKey(desc.getName())) { - logger.error("Dup CubeDesc name '" + desc.getName() + "' on path " + path); - continue; - } - - cubeDescMap.putLocal(desc.getName(), desc); - } - - logger.debug("Loaded " + cubeDescMap.size() + " Cube(s)"); - } - - private CubeDesc loadCubeDesc(String path) throws IOException { - ResourceStore store = getStore(); - logger.debug("Loading CubeDesc " + store.getReadableResourcePath(path)); - - CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER); - - if (StringUtils.isBlank(ndesc.getName())) { - throw new IllegalStateException("CubeDesc name must not be blank"); - } - - ndesc.init(config, getMetadataManager().getAllTablesMap()); - - if (ndesc.getError().isEmpty() == false) { - throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError()); - } - - return ndesc; - } - - /** - * Update CubeDesc with the input. Broadcast the event into cluster - * - * @param desc - * @return - * @throws IOException - */ - public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException { - // Validate CubeDesc - if (desc.getUuid() == null || desc.getName() == null) { - throw new IllegalArgumentException(); - } - String name = desc.getName(); - if (!cubeDescMap.containsKey(name)) { - throw new IllegalArgumentException("CubeDesc '" + name + "' does not exist."); - } - - try { - desc.init(config, getMetadataManager().getAllTablesMap()); - } catch (IllegalStateException e) { - desc.addError(e.getMessage(), true); - return desc; - } catch (IllegalArgumentException e) { - desc.addError(e.getMessage(), true); - return desc; - } - - // Semantic validation - CubeMetadataValidator validator = new CubeMetadataValidator(); - ValidateContext context = validator.validate(desc, true); - if (!context.ifPass()) { - return desc; - } - - desc.setSignature(desc.calculateSignature()); - - // Save Source - String path = desc.getResourcePath(); - getStore().putResource(path, desc, CUBE_DESC_SERIALIZER); - - // Reload the CubeDesc - CubeDesc ndesc = loadCubeDesc(path); - // Here replace the old one - cubeDescMap.put(ndesc.getName(), desc); - - return ndesc; - } - - /** - * Reload CubeDesc from resource store It will be triggered by an desc - * update event. - * - * @param name - * @throws IOException - */ - public CubeDesc reloadCubeDesc(String name) throws IOException { - - // Save Source - String path = CubeDesc.getCubeDescResourcePath(name); - - // Reload the CubeDesc - CubeDesc ndesc = loadCubeDesc(path); - - // Here replace the old one - cubeDescMap.put(ndesc.getName(), ndesc); - return ndesc; - } - public List listAllCubes() { return new ArrayList(cubeMap.values()); } diff --git a/cube/src/main/java/com/kylinolap/cube/cuboid/CuboidCLI.java b/cube/src/main/java/com/kylinolap/cube/cuboid/CuboidCLI.java index 380a073..8a6fc22 100644 --- a/cube/src/main/java/com/kylinolap/cube/cuboid/CuboidCLI.java +++ b/cube/src/main/java/com/kylinolap/cube/cuboid/CuboidCLI.java @@ -22,7 +22,7 @@ import java.util.TreeSet; import com.kylinolap.common.KylinConfig; -import com.kylinolap.cube.CubeManager; +import com.kylinolap.cube.CubeDescManager; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.cube.model.RowKeyDesc; import com.kylinolap.cube.model.RowKeyDesc.AggrGroupMask; @@ -35,10 +35,10 @@ public class CuboidCLI { public static void main(String[] args) throws IOException { - CubeManager metaMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); + CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv()); if ("test".equals(args[0])) { - CubeDesc cubeDesc = metaMgr.getCubeDesc(args[1]); + CubeDesc cubeDesc = cubeDescMgr.getCubeDesc(args[1]); simulateCuboidGeneration(cubeDesc); } } diff --git a/cube/src/test/java/com/kylinolap/cube/CubeDescTest.java b/cube/src/test/java/com/kylinolap/cube/CubeDescTest.java index 45cca7b..2ed9055 100644 --- a/cube/src/test/java/com/kylinolap/cube/CubeDescTest.java +++ b/cube/src/test/java/com/kylinolap/cube/CubeDescTest.java @@ -42,7 +42,7 @@ public void after() throws Exception { @Test public void testSerialize() throws Exception { - CubeDesc desc = CubeManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeDesc desc = CubeDescManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); String str = JsonUtil.writeValueAsIndentString(desc); System.out.println(str); @SuppressWarnings("unused") @@ -52,7 +52,7 @@ public void testSerialize() throws Exception { @Test public void testGetCubeDesc() throws Exception { - CubeDesc cubeDesc = CubeManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeDesc cubeDesc = CubeDescManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc"); Assert.assertNotNull(cubeDesc); } diff --git a/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java b/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java index 87c0734..c9f9b24 100644 --- a/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java +++ b/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java @@ -56,8 +56,8 @@ public void testReloadCache() throws Exception { // clean legacy in case last run failed store.deleteResource("/cube/a_whole_new_cube.json"); - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); cubeManager.createCube("a_whole_new_cube", "default", desc, null); CubeInstance createdCube = cubeManager.getCube("a_whole_new_cube"); @@ -69,7 +69,7 @@ public void testReloadCache() throws Exception { assertEquals(CubeStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus()); } - private CubeManager getCubeManager() { - return CubeManager.getInstance(getTestConfig()); + public CubeDescManager getCubeDescManager() { + return CubeDescManager.getInstance(getTestConfig()); } } diff --git a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java index 0e8b0af..990ea37 100644 --- a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java @@ -67,8 +67,8 @@ public void testCreateAndDrop() throws Exception { // clean legacy in case last run failed store.deleteResource("/cube/a_whole_new_cube.json"); - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("a_whole_new_cube")); @@ -82,7 +82,7 @@ public void testCreateAndDrop() throws Exception { assertNull(CubeManager.getInstance(this.getTestConfig()).getCube("a_whole_new_cube")); } - private CubeManager getCubeManager() { - return CubeManager.getInstance(getTestConfig()); + public CubeDescManager getCubeDescManager() { + return CubeDescManager.getInstance(getTestConfig()); } } diff --git a/cube/src/test/java/com/kylinolap/cube/DictionaryManagerTest.java b/cube/src/test/java/com/kylinolap/cube/DictionaryManagerTest.java index 540e453..e69c9c7 100644 --- a/cube/src/test/java/com/kylinolap/cube/DictionaryManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/DictionaryManagerTest.java @@ -50,7 +50,7 @@ public void after() throws Exception { @Test @Ignore public void basic() throws Exception { - CubeDesc cubeDesc = CubeManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc"); + CubeDesc cubeDesc = CubeDescManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc"); TblColRef col = cubeDesc.findColumnRef("TEST_SITES", "SITE_NAME"); DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null); diff --git a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java index 49f8577..5d0558e 100644 --- a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java +++ b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java @@ -73,8 +73,8 @@ private void createNewCube(CubeDesc cubeDesc) throws IOException { @Test public void testInitialAndAppend() throws ParseException, IOException, CubeIntegrityException { // create a new cube - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); createNewCube(desc); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); @@ -212,8 +212,8 @@ public void testMergeSegments() throws IOException, CubeIntegrityException { @Test public void testNonPartitionedCube() throws ParseException, IOException, CubeIntegrityException { // create a new cube - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_without_slr_desc"); createNewCube(desc); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); @@ -283,8 +283,8 @@ public void testNonPartitionedCube() throws ParseException, IOException, CubeInt @Test(expected = CubeIntegrityException.class) public void testInvalidAppend() throws ParseException, IOException, CubeIntegrityException { // create a new cube - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); createNewCube(desc); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); @@ -332,8 +332,8 @@ public void testInvalidAppend() throws ParseException, IOException, CubeIntegrit @Test public void testInitialAndUpsert() throws ParseException, IOException, CubeIntegrityException { // create a new cube - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); createNewCube(desc); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); @@ -488,8 +488,8 @@ public void testInitialAndUpsert() throws ParseException, IOException, CubeInteg @Test public void testInitialAndUpsert2() throws ParseException, IOException, CubeIntegrityException { // create a new cube - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); createNewCube(desc); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); @@ -603,8 +603,8 @@ public void testInitialAndUpsert2() throws ParseException, IOException, CubeInte @Test(expected = CubeIntegrityException.class) public void testInvalidUpsert() throws IOException, CubeIntegrityException, ParseException { // create a new cube - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); createNewCube(desc); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); @@ -654,8 +654,8 @@ public void testInvalidUpsert() throws IOException, CubeIntegrityException, Pars @Test(expected = CubeIntegrityException.class) public void testInvalidUpsert2() throws IOException, CubeIntegrityException, ParseException { // create a new cube - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); createNewCube(desc); SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); @@ -704,11 +704,7 @@ public void testInvalidUpsert2() throws IOException, CubeIntegrityException, Par System.out.println(JsonUtil.writeValueAsIndentString(cubeInstance)); } - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(getTestConfig()); - } - - private CubeManager getCubeManager() { - return CubeManager.getInstance(getTestConfig()); + public CubeDescManager getCubeDescManager() { + return CubeDescManager.getInstance(getTestConfig()); } } diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java index 7b968e1..0a7db9b 100644 --- a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidSchedulerTest.java @@ -28,7 +28,7 @@ import org.junit.Test; import com.kylinolap.common.util.LocalFileMetadataTestCase; -import com.kylinolap.cube.CubeManager; +import com.kylinolap.cube.CubeDescManager; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.metadata.MetadataManager; @@ -69,15 +69,15 @@ static String toString(Collection cuboids) { } private CubeDesc getTestKylinCubeWithoutSeller() { - return getCubeManager().getCubeDesc("test_kylin_cube_without_slr_desc"); + return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_desc"); } private CubeDesc getTestKylinCubeWithSeller() { - return getCubeManager().getCubeDesc("test_kylin_cube_with_slr_desc"); + return getCubeDescManager().getCubeDesc("test_kylin_cube_with_slr_desc"); } private CubeDesc getTestKylinCubeWithoutSellerLeftJoin() { - return getCubeManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); + return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); } @Test @@ -249,28 +249,28 @@ public void testCuboidGeneration3() { @Test @Ignore public void testCuboidGeneration4() { - CubeDesc cube = getCubeManager().getCubeDesc("geox_trans_mtrc_sd_cube_desc"); + CubeDesc cube = getCubeDescManager().getCubeDesc("geox_trans_mtrc_sd_cube_desc"); CuboidCLI.simulateCuboidGeneration(cube); } @Test @Ignore public void testCuboidGeneration5() { - CubeDesc cube = getCubeManager().getCubeDesc("clsfd_ga_dayweek"); + CubeDesc cube = getCubeDescManager().getCubeDesc("clsfd_ga_dayweek"); CuboidCLI.simulateCuboidGeneration(cube); } @Test @Ignore public void testCuboidGeneration6() { - CubeDesc cube = getCubeManager().getCubeDesc("clsfd_ga_day"); + CubeDesc cube = getCubeDescManager().getCubeDesc("clsfd_ga_day"); CuboidCLI.simulateCuboidGeneration(cube); } @Test @Ignore public void testCuboidGeneration7() { - CubeDesc cube = getCubeManager().getCubeDesc("clsfd_ga_week"); + CubeDesc cube = getCubeDescManager().getCubeDesc("clsfd_ga_week"); CuboidCLI.simulateCuboidGeneration(cube); } @@ -297,12 +297,8 @@ private String sortToString(Collection longs) { return copy.toString(); } - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(getTestConfig()); - } - - private CubeManager getCubeManager() { - return CubeManager.getInstance(getTestConfig()); + public CubeDescManager getCubeDescManager() { + return CubeDescManager.getInstance(getTestConfig()); } private void printCount(int[] counts) { diff --git a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java index 6c0c816..edd2a11 100644 --- a/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java +++ b/cube/src/test/java/com/kylinolap/cube/cuboid/CuboidTest.java @@ -22,7 +22,7 @@ import org.junit.Test; import com.kylinolap.common.util.LocalFileMetadataTestCase; -import com.kylinolap.cube.CubeManager; +import com.kylinolap.cube.CubeDescManager; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.metadata.MetadataManager; @@ -35,25 +35,20 @@ private long toLong(String bin) { return Long.parseLong(bin, 2); } - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(getTestConfig()); + public CubeDescManager getCubeDescManager() { + return CubeDescManager.getInstance(getTestConfig()); } - private CubeManager getCubeManager() { - return CubeManager.getInstance(getTestConfig()); - } - - private CubeDesc getTestKylinCubeII() { - return getCubeManager().getCubeDesc("test_kylin_cube_ii"); + return getCubeDescManager().getCubeDesc("test_kylin_cube_ii"); } private CubeDesc getTestKylinCubeWithoutSeller() { - return getCubeManager().getCubeDesc("test_kylin_cube_without_slr_desc"); + return getCubeDescManager().getCubeDesc("test_kylin_cube_without_slr_desc"); } private CubeDesc getTestKylinCubeWithSeller() { - return getCubeManager().getCubeDesc("test_kylin_cube_with_slr_desc"); + return getCubeDescManager().getCubeDesc("test_kylin_cube_with_slr_desc"); } @Before diff --git a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java b/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java index 0d14862..de54438 100644 --- a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java @@ -27,6 +27,7 @@ import com.kylinolap.common.persistence.ResourceStore; import com.kylinolap.common.util.JsonUtil; import com.kylinolap.common.util.LocalFileMetadataTestCase; +import com.kylinolap.cube.CubeDescManager; import com.kylinolap.cube.CubeInstance; import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.model.CubeDesc; @@ -71,8 +72,8 @@ public void testNewProject() throws Exception { // clean legacy in case last run failed store.deleteResource("/cube/cube_in_alien_project.json"); - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("cube_in_alien_project", "alien", desc, null); assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("cube_in_alien_project")); assertTrue(ProjectManager.getInstance(getTestConfig()).listAllCubes("alien").contains(createdCube)); @@ -115,8 +116,8 @@ public void testExistingProject() throws Exception { // clean legacy in case last run failed store.deleteResource("/cube/new_cube_in_default.json"); - CubeManager metaMgr = getCubeManager(); - CubeDesc desc = metaMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("new_cube_in_default")); @@ -166,4 +167,8 @@ private MetadataManager getMetadataManager() { private CubeManager getCubeManager() { return CubeManager.getInstance(getTestConfig()); } + + public CubeDescManager getCubeDescManager() { + return CubeDescManager.getInstance(getTestConfig()); + } } diff --git a/server/src/main/java/com/kylinolap/rest/service/BasicService.java b/server/src/main/java/com/kylinolap/rest/service/BasicService.java index b76da75..d202313 100644 --- a/server/src/main/java/com/kylinolap/rest/service/BasicService.java +++ b/server/src/main/java/com/kylinolap/rest/service/BasicService.java @@ -40,6 +40,7 @@ import com.google.common.io.Files; import com.kylinolap.common.KylinConfig; +import com.kylinolap.cube.CubeDescManager; import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; @@ -154,6 +155,10 @@ public CubeManager getCubeManager() { return CubeManager.getInstance(getConfig()); } + public CubeDescManager getCubeDescManager() { + return CubeDescManager.getInstance(getConfig()); + } + public ProjectManager getProjectManager() { return ProjectManager.getInstance(getConfig()); } diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java index fa3c181..4c3eea9 100644 --- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java +++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java @@ -163,10 +163,10 @@ public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeD CubeDesc createdDesc = null; CubeInstance createdCube = null; - createdDesc = getCubeManager().createCubeDesc(desc); + createdDesc = getCubeDescManager().createCubeDesc(desc); if (!createdDesc.getError().isEmpty()) { - getCubeManager().removeCubeDesc(createdDesc); + getCubeDescManager().removeCubeDesc(createdDesc); throw new InternalErrorException(createdDesc.getError().get(0)); } @@ -174,7 +174,7 @@ public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeD int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc); logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids"); }catch(Exception e){ - getCubeManager().removeCubeDesc(createdDesc); + getCubeDescManager().removeCubeDesc(createdDesc); throw new InternalErrorException("Failed to deal with the request.", e); } @@ -201,7 +201,7 @@ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newPr this.releaseAllSegments(cube); } - CubeDesc updatedCubeDesc = getCubeManager().updateCubeDesc(desc); + CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc); int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc); logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids"); @@ -552,7 +552,7 @@ private static void writeResource(InputStream source, KylinConfig dstConfig, Str public void updateCubeNotifyList(CubeInstance cube, List notifyList) throws IOException, CubeIntegrityException { CubeDesc desc = cube.getDescriptor(); desc.setNotifyList(notifyList); - getCubeManager().updateCubeDesc(desc); + getCubeDescManager().updateCubeDesc(desc); } public CubeInstance rebuildLookupSnapshot(String cubeName, String segmentName, String lookupTable) throws IOException { From 53c6067fd7d1c53c34bd3a33bca0449a5422c571 Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Mon, 1 Dec 2014 16:05:46 +0800 Subject: [PATCH 2/3] Update test case data with the new metadata structure --- .../java/com/kylinolap/cube/CubeDescManager.java | 2 +- .../java/com/kylinolap/cube/model/CubeDesc.java | 3 +- .../cube_desc/test_kylin_cube_with_slr_desc.json | 33 ++--------------- .../model_desc/test_kylin_with_slr_model_desc.json | 43 ++++++++++++++++++++++ 4 files changed, 49 insertions(+), 32 deletions(-) create mode 100644 examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json diff --git a/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java b/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java index 033c2d8..91f4e2d 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeDescManager.java @@ -91,7 +91,7 @@ public static synchronized void removeInstance(KylinConfig config) { private CubeDescManager(KylinConfig config) throws IOException { logger.info("Initializing CubeDescManager with config " + config); this.config = config; - + reloadAllCubeDesc(); } public CubeDesc getCubeDesc(String name) { diff --git a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java index aedb80f..f8f3f8d 100644 --- a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java +++ b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java @@ -479,6 +479,7 @@ public String calculateSignature() { public void init(KylinConfig config, Map tables) { this.errors.clear(); this.config = config; + this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); for (DimensionDesc dim : dimensions) { dim.init(this, tables); @@ -489,8 +490,6 @@ public void init(KylinConfig config, Map tables) { initJoinColumns(tables); initDimensionColumns(tables); initMeasureColumns(tables); - - //TODO create the DataModel object from MetadataManager and then set here; rowkey.init(this); if (hbaseMapping != null) { diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json index e7f8d79..8bc0b17 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json @@ -1,7 +1,7 @@ { "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92", "name": "test_kylin_cube_with_slr_desc", - "fact_table": "test_kylin_fact", + "model_name": "test_kylin_with_slr_model_desc" "filter_condition": null, "cube_partition_desc": { "partition_date_column": "test_kylin_fact.cal_dt", @@ -12,15 +12,9 @@ { "id": "1", "name": "cal_dt", - "datatype": "date", "table": "test_cal_dt", "column": "{FK}", - "derived": ["week_beg_dt"], - "join": { - "type": "inner", - "primary_key": ["cal_dt"], - "foreign_key": ["cal_dt"] - } + "derived": ["week_beg_dt"] }, { "id": "2", @@ -28,11 +22,6 @@ "table": "test_category_groupings", "column": "{FK}", "derived": ["USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER"], - "join": { - "type": "inner", - "primary_key": ["leaf_categ_id", "site_id"], - "foreign_key": ["leaf_categ_id", "lstg_site_id"] - }, "hierarchy": [ { "level": "1", @@ -51,40 +40,26 @@ { "id": "3", "name": "lstg_format_name", - "datatype": "string", "table": "test_kylin_fact", "column": "lstg_format_name" }, { "id": "4", "name": "site_id", - "datatype": "string", "table": "test_sites", "column": "{FK}", - "derived": ["site_name", "cre_user"], - "join": { - "type": "inner", - "primary_key": ["site_id"], - "foreign_key": ["lstg_site_id"] - } + "derived": ["site_name", "cre_user"] }, { "id": "5", "name": "seller_type_cd", - "datatype": "string", "table": "test_seller_type_dim", "column": "{FK}", - "derived": ["seller_type_desc"], - "join": { - "type": "inner", - "primary_key": ["seller_type_cd"], - "foreign_key": ["slr_segment_cd"] - } + "derived": ["seller_type_desc"] }, { "id": "6", "name": "seller_id", - "datatype": "string", "table": "test_kylin_fact", "column": "seller_id" } diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json new file mode 100644 index 0000000..7209e88 --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json @@ -0,0 +1,43 @@ +{ + "uuid": "a24ca905-1fc6-4f67-985c-341fewaeafd92", + "name": "test_kylin_with_slr_model_desc", + "fact_table": "test_kylin_fact", + "lookups": [ + { + "id": "1", + "table": "test_cal_dt", + "join": { + "type": "inner", + "primary_key": ["cal_dt"], + "foreign_key": ["cal_dt"] + } + }, + { + "id": "2", + "table": "test_category_groupings", + "join": { + "type": "inner", + "primary_key": ["leaf_categ_id", "site_id"], + "foreign_key": ["leaf_categ_id", "lstg_site_id"] + } + }, + { + "id": "3", + "table": "test_sites", + "join": { + "type": "inner", + "primary_key": ["site_id"], + "foreign_key": ["lstg_site_id"] + } + }, + { + "id": "4", + "table": "test_seller_type_dim", + "join": { + "type": "inner", + "primary_key": ["seller_type_cd"], + "foreign_key": ["slr_segment_cd"] + } + } + ] +} From a0d6310c38516178e6b26aa52e24e2b4197dd862 Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Tue, 2 Dec 2014 17:49:57 +0800 Subject: [PATCH 3/3] This commit includes the following changes: 1. Code change for the metadata model structure change; 2. Update the example data, separating cube descriptor to cube + model descriptor; 3. Update some test cases. --- .../common/persistence/ResourceStore.java | 1 + .../com/kylinolap/cube/CubeSegmentValidator.java | 8 +- .../kylinolap/cube/dataGen/FactTableGenerator.java | 6 +- .../java/com/kylinolap/cube/model/CubeDesc.java | 97 ++--------- .../com/kylinolap/cube/model/DimensionDesc.java | 85 +++++---- cube/src/test/resources/data/TEST1_desc.json | 192 +++++++++++++++++++++ cube/src/test/resources/data/TEST2_desc.json | 192 +++++++++++++++++++++ cube/src/test/resources/data/TEST3_desc.json | 192 +++++++++++++++++++++ .../localmeta/cube_desc/test_kylin_cube_ii.json | 82 ++------- .../cube_desc/test_kylin_cube_with_slr_desc.json | 41 +---- .../test_kylin_cube_with_slr_left_join_desc.json | 82 ++------- .../test_kylin_cube_without_slr_desc.json | 84 ++------- ...test_kylin_cube_without_slr_left_join_desc.json | 85 ++------- .../model_desc/test_kylin_ii_model_desc.json | 38 ++++ .../test_kylin_with_slr_left_join_model_desc.json | 46 +++++ .../model_desc/test_kylin_with_slr_model_desc.json | 5 - ...est_kylin_without_slr_left_join_model_desc.json | 56 ++++++ .../test_kylin_without_slr_model_desc.json | 56 ++++++ .../com/kylinolap/metadata/MetadataManager.java | 121 +++++++++++-- .../kylinolap/metadata/model/DataModelDesc.java | 113 +++++++++++- metadata/src/test/resources/data/TEST1_desc.json | 192 --------------------- metadata/src/test/resources/data/TEST2_desc.json | 192 --------------------- metadata/src/test/resources/data/TEST3_desc.json | 192 --------------------- .../endpoint/EndpoindAggregationTest.java | 19 +- 24 files changed, 1132 insertions(+), 1045 deletions(-) create mode 100644 cube/src/test/resources/data/TEST1_desc.json create mode 100644 cube/src/test/resources/data/TEST2_desc.json create mode 100644 cube/src/test/resources/data/TEST3_desc.json create mode 100644 examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json create mode 100644 examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json create mode 100644 examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json create mode 100644 examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json delete mode 100644 metadata/src/test/resources/data/TEST1_desc.json delete mode 100644 metadata/src/test/resources/data/TEST2_desc.json delete mode 100644 metadata/src/test/resources/data/TEST3_desc.json diff --git a/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java b/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java index 36abfdd..e920d4d 100644 --- a/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java +++ b/common/src/main/java/com/kylinolap/common/persistence/ResourceStore.java @@ -37,6 +37,7 @@ public static final String CUBE_RESOURCE_ROOT = "/cube"; public static final String CUBE_DESC_RESOURCE_ROOT = "/cube_desc"; + public static final String DATA_MODEL_DESC_RESOURCE_ROOT = "/model_desc"; public static final String DICT_RESOURCE_ROOT = "/dict"; public static final String IIDESC_RESOURCE_ROOT = "/invertedindex_desc"; public static final String JOB_PATH_ROOT = "/job"; diff --git a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java index 3f50032..7d83e2e 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java @@ -93,9 +93,11 @@ private void checkLoopTableConsistency(CubeInstance cube, List newS for (TblColRef col : dim.getColumnRefs()) { // include those dictionaries that do not need mergning try { - String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; - if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col) && !cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) { - cols.add(col); + if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { + String dictTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; + if (!cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) { + cols.add(col); + } } } catch (IOException e) { throw new CubeIntegrityException("checkLoopTableConsistency not passed when allocating a new segment."); diff --git a/cube/src/main/java/com/kylinolap/cube/dataGen/FactTableGenerator.java b/cube/src/main/java/com/kylinolap/cube/dataGen/FactTableGenerator.java index 49b26eb..7781e61 100644 --- a/cube/src/main/java/com/kylinolap/cube/dataGen/FactTableGenerator.java +++ b/cube/src/main/java/com/kylinolap/cube/dataGen/FactTableGenerator.java @@ -304,8 +304,10 @@ private String generate() throws Exception { JoinDesc jDesc = dim.getJoin(); if (jDesc == null) { // column on fact table used directly as a dimension - if (!factTableCol2LookupCol.containsKey(dim.getColumn())) - usedCols.add(dim.getColumn()); + for (String aColumn : dim.getColumn()) { + if (!factTableCol2LookupCol.containsKey(aColumn)) + usedCols.add(aColumn); + } } } diff --git a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java index f8f3f8d..56745ad 100644 --- a/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java +++ b/cube/src/main/java/com/kylinolap/cube/model/CubeDesc.java @@ -203,28 +203,6 @@ public DimensionDesc findDimensionByName(String dimName) { return null; } -// public TblColRef findPKByFK(TblColRef fk) { -// assert isFactTable(fk.getTable()); -// -// TblColRef candidate = null; -// -// for (DimensionDesc dim : dimensions) { -// JoinDesc join = dim.getJoin(); -// if (join == null) -// continue; -// -// int find = ArrayUtils.indexOf(join.getForeignKeyColumns(), fk); -// if (find >= 0) { -// candidate = join.getPrimaryKeyColumns()[find]; -// if (join.getForeignKeyColumns().length == 1) { // is single -// // column join? -// break; -// } -// } -// } -// return candidate; -// } - /** * Get all functions from each measure. * @@ -479,15 +457,21 @@ public String calculateSignature() { public void init(KylinConfig config, Map tables) { this.errors.clear(); this.config = config; + if(this.modelName == null || this.modelName.length() ==0) { + this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); + } + this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); + + if(this.model == null) { + this.addError("No data model found with name '" + modelName + "'."); + } for (DimensionDesc dim : dimensions) { dim.init(this, tables); } sortDimAndMeasure(); - - initJoinColumns(tables); initDimensionColumns(tables); initMeasureColumns(tables); @@ -520,16 +504,19 @@ private void initDimensionColumns(Map tables) { // dimension column if (dim.getColumn() != null) { - if ("{FK}".equals(dim.getColumn())) { + //if ("{FK}".equals(dim.getColumn())) { + if (join != null) { for (TblColRef ref : join.getForeignKeyColumns()) { TblColRef inited = initDimensionColRef(ref); dimColList.add(inited); hostColList.add(inited); } } else { - TblColRef ref = initDimensionColRef(dimTable, dim.getColumn()); - dimColList.add(ref); - hostColList.add(ref); + for (String aColumn : dim.getColumn()) { + TblColRef ref = initDimensionColRef(dimTable, aColumn); + dimColList.add(ref); + hostColList.add(ref); + } } } // hierarchy columns @@ -649,60 +636,6 @@ private TblColRef initDimensionColRef(TblColRef ref) { return ref; } - private void initJoinColumns(Map tables) { - // join columns may or may not present in cube; - // here we don't modify 'allColumns' and 'dimensionColumns'; - // initDimensionColumns() will do the update - for (DimensionDesc dim : dimensions) { - TableDesc dimTable = tables.get(dim.getTable()); - - JoinDesc join = dim.getJoin(); - if (join == null) - continue; - - // primary key - String[] pks = join.getPrimaryKey(); - TblColRef[] pkCols = new TblColRef[pks.length]; - for (int i = 0; i < pks.length; i++) { - ColumnDesc col = dimTable.findColumnByName(pks[i]); - if (col == null) { - addError("Can't find column " + pks[i] + " in table " + dimTable.getName()); - } - TblColRef colRef = new TblColRef(col); - pks[i] = colRef.getName(); - pkCols[i] = colRef; - } - join.setPrimaryKeyColumns(pkCols); - // foreign key - TableDesc factTable = tables.get(this.getFactTable()); - if (factTable == null) { - addError("Fact table does not exist:" + this.getFactTable()); - } - String[] fks = join.getForeignKey(); - TblColRef[] fkCols = new TblColRef[fks.length]; - for (int i = 0; i < fks.length; i++) { - ColumnDesc col = factTable.findColumnByName(fks[i]); - if (col == null) { - addError("Can't find column " + fks[i] + " in table " + this.getFactTable()); - } - TblColRef colRef = new TblColRef(col); - fks[i] = colRef.getName(); - fkCols[i] = colRef; - } - join.setForeignKeyColumns(fkCols); - // Validate join in dimension - if (pkCols.length != fkCols.length) { - addError("Primary keys(" + dim.getTable() + ")" + Arrays.toString(pks) + " are not consistent with Foreign keys(" + this.getFactTable() + ") " + Arrays.toString(fks)); - } - for (int i = 0; i < fkCols.length; i++) { - if (!fkCols[i].getDatatype().equals(pkCols[i].getDatatype())) { - addError("Primary key " + dim.getTable() + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with Foreign key " + this.getFactTable() + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype()); - } - } - - } - } - private void initMeasureColumns(Map tables) { if (measures == null || measures.isEmpty()) { return; diff --git a/cube/src/main/java/com/kylinolap/cube/model/DimensionDesc.java b/cube/src/main/java/com/kylinolap/cube/model/DimensionDesc.java index 8c43fd7..c041037 100644 --- a/cube/src/main/java/com/kylinolap/cube/model/DimensionDesc.java +++ b/cube/src/main/java/com/kylinolap/cube/model/DimensionDesc.java @@ -15,12 +15,15 @@ */ package com.kylinolap.cube.model; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import com.kylinolap.common.util.StringSplitter; import com.kylinolap.common.util.StringUtil; import com.kylinolap.metadata.model.JoinDesc; import com.kylinolap.metadata.model.LookupDesc; @@ -39,11 +42,13 @@ @JsonProperty("name") private String name; private JoinDesc join; - @JsonProperty("hierarchy") private HierarchyDesc[] hierarchy; + @JsonProperty("hierarchy") + private boolean isHierarchy; private String table; + private String database; @JsonProperty("column") - private String column; + private String[] column; @JsonProperty("derived") private String[] derived; @@ -95,14 +100,12 @@ public void setColumnRefs(TblColRef[] colRefs) { this.columnRefs = colRefs; } - public String getColumn() { + public String[] getColumn() { return this.column; } - public void setColumn(String column) { + public void setColumn(String[] column) { this.column = column; - if (this.column != null) - this.column = this.column.toUpperCase(); } public HierarchyDesc[] getHierarchy() { @@ -128,6 +131,7 @@ public void setDerived(String[] derived) { public void setDerivedColRefs(TblColRef[] derivedColRefs) { this.derivedColRefs = derivedColRefs; } + @Override public boolean equals(Object o) { @@ -155,47 +159,68 @@ public int hashCode() { @Override public String toString() { - return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + column + ", derived=" + Arrays.toString(derived) + "]"; + return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]"; } public void init(CubeDesc cubeDesc, Map tables) { if (name != null) name = name.toUpperCase(); - if (column != null) - column = column.toUpperCase(); - // parse 'column' to get the table name; if table name is not appeared, use fact table; - String[] splits = this.column.split("."); - if (splits.length > 1) { - table = splits[splits.length - 2].toUpperCase(); - } else { - table = cubeDesc.getFactTable().toUpperCase(); + this.table = null; + this.database = null; + this.join = null; + + for(int i=0, n = this.column.length; i 1) { + String thisTable = splits[splits.length - 2].toUpperCase(); + if(table == null) { + table = thisTable; + } else if (thisTable != null && !table.equalsIgnoreCase(thisTable)) { + throw new IllegalStateException("One dimension can only refer to the columns on the same table."); + } + + if (database == null && splits.length > 2) { + database = splits[splits.length - 3].toUpperCase(); + } + + this.column[i] = splits[splits.length - 1].toUpperCase(); + } else { + // if no table specified, assume it is on fact table + table = cubeDesc.getFactTable(); + } } - + TableDesc tableDesc = tables.get(table); if (tableDesc == null) throw new IllegalStateException("Can't find table " + table + " on dimension " + name); + + for(LookupDesc lookup : cubeDesc.getModel().getLookups()) { + if(lookup.getTable().equalsIgnoreCase(table)) { + this.join = lookup.getJoin(); + break; + } + } - boolean isOnFactTable = table.equalsIgnoreCase(cubeDesc.getFactTable()); + if (isHierarchy && this.column.length > 0) { + List hierarchyList = new ArrayList(3); + for (int i = 0, n = this.column.length; i < n; i++) { + String aColumn = this.column[i]; + HierarchyDesc aHierarchy = new HierarchyDesc(); + aHierarchy.setLevel(String.valueOf(i + 1)); + aHierarchy.setColumn(aColumn); + hierarchyList.add(aHierarchy); + } + + this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]); + } if (hierarchy != null && hierarchy.length == 0) hierarchy = null; if (derived != null && derived.length == 0) derived = null; - if(!isOnFactTable) { - for(LookupDesc lookup: cubeDesc.getModel().getLookups()) { - if(lookup.getTable().equals(table)) { - join = lookup.getJoin(); - break; - } - } - } - - if (join != null) { - StringUtil.toUpperCaseArray(join.getForeignKey(), join.getForeignKey()); - StringUtil.toUpperCaseArray(join.getPrimaryKey(), join.getPrimaryKey()); - } if (hierarchy != null) { for (HierarchyDesc h : hierarchy) diff --git a/cube/src/test/resources/data/TEST1_desc.json b/cube/src/test/resources/data/TEST1_desc.json new file mode 100644 index 0000000..175b105 --- /dev/null +++ b/cube/src/test/resources/data/TEST1_desc.json @@ -0,0 +1,192 @@ +{ + "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b", + "last_modified" : 1401429176099, + "name" : "TEST1_desc", + "fact_table" : "TEST_KYLIN_FACT", + "dimensions" : [ { + "id" : 1, + "name" : "CAL_DT", + "join" : { + "type" : "inner", + "primary_key" : [ "CAL_DT" ], + "foreign_key" : [ "CAL_DT" ] + }, + "hierarchy" : null, + "table" : "TEST_CAL_DT", + "column" : "CAL_DT", + "datatype" : "date", + "derived" : [ "WEEK_BEG_DT" ] + }, { + "id" : 2, + "name" : "CATEGORY", + "join" : { + "type" : "inner", + "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], + "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] + }, + "hierarchy" : [ { + "level" : "1", + "column" : "META_CATEG_NAME" + }, { + "level" : "2", + "column" : "CATEG_LVL2_NAME" + }, { + "level" : "3", + "column" : "CATEG_LVL3_NAME" + } ], + "table" : "TEST_CATEGORY_GROUPINGS", + "column" : null, + "datatype" : null, + "derived" : null + }, { + "id" : 3, + "name" : "LSTG_FORMAT_NAME", + "join" : null, + "hierarchy" : null, + "table" : "TEST_KYLIN_FACT", + "column" : "LSTG_FORMAT_NAME", + "datatype" : "string", + "derived" : null + }, { + "id" : 4, + "name" : "SITE_ID", + "join" : { + "type" : "inner", + "primary_key" : [ "SITE_ID" ], + "foreign_key" : [ "LSTG_SITE_ID" ] + }, + "hierarchy" : null, + "table" : "TEST_SITES", + "column" : "SITE_ID", + "datatype" : "string", + "derived" : [ "SITE_NAME", "CRE_USER" ] + }, { + "id" : 5, + "name" : "SELLER_TYPE_CD", + "join" : { + "type" : "inner", + "primary_key" : [ "SELLER_TYPE_CD" ], + "foreign_key" : [ "SLR_SEGMENT_CD" ] + }, + "hierarchy" : null, + "table" : "TEST_SELLER_TYPE_DIM", + "column" : "SELLER_TYPE_CD", + "datatype" : "string", + "derived" : [ "SELLER_TYPE_DESC" ] + } ], + "measures" : [ { + "id" : 1, + "name" : "GMV_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal" + } + }, { + "id" : 2, + "name" : "GMV_MIN", + "function" : { + "expression" : "MIN", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal" + } + }, { + "id" : 3, + "name" : "GMV_MAX", + "function" : { + "expression" : "MAX", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal" + } + }, { + "id" : 4, + "name" : "TRANS_CNT", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1" + }, + "returntype" : "long" + } + }, { + "id" : 5, + "name" : "SELLER_CNT", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "SELLER_ID" + }, + "returntype" : "hllc10" + } + }, { + "id" : 6, + "name" : "SELLER_FORMAT_CNT", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "LSTG_FORMAT_NAME,SELLER_ID" + }, + "returntype" : "hllc10" + } + } ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "CAL_DT", + "length" : 0, + "dictionary" : "date(yyyy-mm-dd)", + "mandatory" : true + }, { + "column" : "META_CATEG_NAME", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "CATEG_LVL2_NAME", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "CATEG_LVL3_NAME", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "LSTG_FORMAT_NAME", + "length" : 12, + "dictionary" : null, + "mandatory" : false + }, { + "column" : "SITE_ID", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "SELLER_TYPE_CD", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + } ], + "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CATEG_LVL2_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD" ] ] + }, + "hbase_mapping" : { + "column_family" : [ { + "name" : "F1", + "columns" : [ { + "qualifier" : "M", + "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ] + } ] + } ] + } +} \ No newline at end of file diff --git a/cube/src/test/resources/data/TEST2_desc.json b/cube/src/test/resources/data/TEST2_desc.json new file mode 100644 index 0000000..81bde27 --- /dev/null +++ b/cube/src/test/resources/data/TEST2_desc.json @@ -0,0 +1,192 @@ +{ + "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b", + "last_modified" : 1401429176099, + "name" : "TEST2_desc", + "fact_table" : "TEST_KYLIN_FACT", + "dimensions" : [ { + "id" : 1, + "name" : "CAL_DT", + "join" : { + "type" : "inner", + "primary_key" : [ "CAL_DT" ], + "foreign_key" : [ "CAL_DT" ] + }, + "hierarchy" : null, + "table" : "TEST_CAL_DT", + "column" : "CAL_DT", + "datatype" : "date", + "derived" : [ "WEEK_BEG_DT" ] + }, { + "id" : 2, + "name" : "CATEGORY", + "join" : { + "type" : "inner", + "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], + "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] + }, + "hierarchy" : [ { + "level" : "1", + "column" : "META_CATEG_NAME" + }, { + "level" : "2", + "column" : "CATEG_LVL2_NAME" + }, { + "level" : "3", + "column" : "CATEG_LVL3_NAME" + } ], + "table" : "TEST_CATEGORY_GROUPINGS", + "column" : null, + "datatype" : null, + "derived" : null + }, { + "id" : 3, + "name" : "LSTG_FORMAT_NAME", + "join" : null, + "hierarchy" : null, + "table" : "TEST_KYLIN_FACT", + "column" : "LSTG_FORMAT_NAME", + "datatype" : "string", + "derived" : null + }, { + "id" : 4, + "name" : "SITE_ID", + "join" : { + "type" : "inner", + "primary_key" : [ "SITE_ID" ], + "foreign_key" : [ "LSTG_SITE_ID" ] + }, + "hierarchy" : null, + "table" : "TEST_SITES", + "column" : "SITE_ID", + "datatype" : "string", + "derived" : [ "SITE_NAME", "CRE_USER" ] + }, { + "id" : 5, + "name" : "SELLER_TYPE_CD", + "join" : { + "type" : "inner", + "primary_key" : [ "SELLER_TYPE_CD" ], + "foreign_key" : [ "SLR_SEGMENT_CD" ] + }, + "hierarchy" : null, + "table" : "TEST_SELLER_TYPE_DIM", + "column" : "SELLER_TYPE_CD", + "datatype" : "string", + "derived" : [ "SELLER_TYPE_DESC" ] + } ], + "measures" : [ { + "id" : 1, + "name" : "GMV_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal" + } + }, { + "id" : 2, + "name" : "GMV_MIN", + "function" : { + "expression" : "MIN", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal" + } + }, { + "id" : 3, + "name" : "GMV_MAX", + "function" : { + "expression" : "MAX", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal" + } + }, { + "id" : 4, + "name" : "TRANS_CNT", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1" + }, + "returntype" : "long" + } + }, { + "id" : 5, + "name" : "SELLER_CNT", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "SELLER_ID" + }, + "returntype" : "hllc10" + } + }, { + "id" : 6, + "name" : "SELLER_FORMAT_CNT", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "LSTG_FORMAT_NAME,SELLER_ID" + }, + "returntype" : "hllc10" + } + } ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "CAL_DT", + "length" : 0, + "dictionary" : "date(yyyy-mm-dd)", + "mandatory" : true + }, { + "column" : "META_CATEG_NAME", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "CATEG_LVL2_NAME", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "CATEG_LVL3_NAME", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "LSTG_FORMAT_NAME", + "length" : 12, + "dictionary" : null, + "mandatory" : false + }, { + "column" : "SITE_ID", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "SELLER_TYPE_CD", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + } ], + "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD", "CATEG_LVL2_NAME" ] ] + }, + "hbase_mapping" : { + "column_family" : [ { + "name" : "F1", + "columns" : [ { + "qualifier" : "M", + "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ] + } ] + } ] + } +} \ No newline at end of file diff --git a/cube/src/test/resources/data/TEST3_desc.json b/cube/src/test/resources/data/TEST3_desc.json new file mode 100644 index 0000000..df4d60b --- /dev/null +++ b/cube/src/test/resources/data/TEST3_desc.json @@ -0,0 +1,192 @@ +{ + "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b", + "last_modified" : 1401429176099, + "name" : "TEST1_desc", + "fact_table" : "TEST_KYLIN_FACT", + "dimensions" : [ { + "id" : 1, + "name" : "CAL_DT", + "join" : { + "type" : "inner", + "primary_key" : [ "CAL_DT" ], + "foreign_key" : [ "CAL_DT" ] + }, + "hierarchy" : null, + "table" : "TEST_CAL_DT", + "column" : "CAL_DT", + "datatype" : "date", + "derived" : [ "WEEK_BEG_DT" ] + }, { + "id" : 2, + "name" : "CATEGORY", + "join" : { + "type" : "inner", + "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], + "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] + }, + "hierarchy" : [ { + "level" : "1", + "column" : "META_CATEG_NAME" + }, { + "level" : "2", + "column" : "CATEG_LVL2_NAME" + }, { + "level" : "3", + "column" : "CATEG_LVL3_NAME" + } ], + "table" : "TEST_CATEGORY_GROUPINGS", + "column" : null, + "datatype" : null, + "derived" : null + }, { + "id" : 3, + "name" : "LSTG_FORMAT_NAME", + "join" : null, + "hierarchy" : null, + "table" : "TEST_KYLIN_FACT", + "column" : "LSTG_FORMAT_NAME", + "datatype" : "string", + "derived" : null + }, { + "id" : 4, + "name" : "SITE_ID", + "join" : { + "type" : "inner", + "primary_key" : [ "SITE_ID" ], + "foreign_key" : [ "LSTG_SITE_ID" ] + }, + "hierarchy" : null, + "table" : "TEST_SITES", + "column" : "SITE_ID", + "datatype" : "string", + "derived" : [ "SITE_NAME", "CRE_USER" ] + }, { + "id" : 5, + "name" : "SELLER_TYPE_CD", + "join" : { + "type" : "inner", + "primary_key" : [ "SELLER_TYPE_CD" ], + "foreign_key" : [ "SLR_SEGMENT_CD" ] + }, + "hierarchy" : null, + "table" : "TEST_SELLER_TYPE_DIM", + "column" : "SELLER_TYPE_CD", + "datatype" : "string", + "derived" : [ "SELLER_TYPE_DESC" ] + } ], + "measures" : [ { + "id" : 1, + "name" : "GMV_SUM", + "function" : { + "expression" : "SUM", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal" + } + }, { + "id" : 2, + "name" : "GMV_MIN", + "function" : { + "expression" : "MIN", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal" + } + }, { + "id" : 3, + "name" : "GMV_MAX", + "function" : { + "expression" : "MAX", + "parameter" : { + "type" : "column", + "value" : "PRICE" + }, + "returntype" : "decimal" + } + }, { + "id" : 4, + "name" : "TRANS_CNT", + "function" : { + "expression" : "COUNT", + "parameter" : { + "type" : "constant", + "value" : "1" + }, + "returntype" : "long" + } + }, { + "id" : 5, + "name" : "SELLER_CNT", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "SELLER_ID" + }, + "returntype" : "hllc10" + } + }, { + "id" : 6, + "name" : "SELLER_FORMAT_CNT", + "function" : { + "expression" : "COUNT_DISTINCT", + "parameter" : { + "type" : "column", + "value" : "LSTG_FORMAT_NAME,SELLER_ID" + }, + "returntype" : "hllc10" + } + } ], + "rowkey" : { + "rowkey_columns" : [ { + "column" : "CAL_DT", + "length" : 10, + "dictionary" : "date(yyyy-mm-dd)", + "mandatory" : false + }, { + "column" : "META_CATEG_NAME", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "CATEG_LVL2_NAME", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "CATEG_LVL3_NAME", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "LSTG_FORMAT_NAME", + "length" : 12, + "dictionary" : null, + "mandatory" : false + }, { + "column" : "SITE_ID", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + }, { + "column" : "SELLER_TYPE_CD", + "length" : 0, + "dictionary" : "string", + "mandatory" : false + } ], + "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CATEG_LVL2_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD" ] ] + }, + "hbase_mapping" : { + "column_family" : [ { + "name" : "F1", + "columns" : [ { + "qualifier" : "M", + "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ] + } ] + } ] + } +} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_ii.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_ii.json index f79dcf1..87ad1b7 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_ii.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_ii.json @@ -1,103 +1,42 @@ { "uuid" : "ac13878c-8767-4454-8aab-1007e274fa23", "name" : "test_kylin_cube_ii", + "model_name" : "test_kylin_ii_model_desc", "description" : null, "dimensions" : [ { - "id" : 1, "name" : "TRANS_ID", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : "TRANS_ID", - "datatype" : "bigint", + "column" : ["TEST_KYLIN_FACT.TRANS_ID"], "derived" : null }, { - "id" : 2, "name" : "CAL_DT", - "join" : { - "type" : "left", - "primary_key" : [ "CAL_DT" ], - "foreign_key" : [ "CAL_DT" ] - }, - "hierarchy" : null, - "table" : "TEST_CAL_DT", - "column" : "{FK}", - "datatype" : "date", + "column" : ["TEST_CAL_DT.CAL_DT"], "derived" : [ "CAL_DT", "WEEK_BEG_DT", "YEAR_BEG_DT", "QTR_BEG_DT", "MONTH_BEG_DT" ] }, { - "id" : 3, "name" : "LSTG_FORMAT_NAME", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : "LSTG_FORMAT_NAME", - "datatype" : "string", - "derived" : null + "column" : ["TEST_KYLIN_FACT.LSTG_FORMAT_NAME"] }, { - "id" : 4, "name" : "LEAF_CATEG_ID", - "join" : { - "type" : "left", - "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], - "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] - }, - "hierarchy" : null, - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : "{FK}", - "datatype" : null, + "column" : ["TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID", "TEST_CATEGORY_GROUPINGS.SITE_ID"], "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] }, { - "id" : 5, "name" : "LSTG_SITE_ID", - "join" : { - "type" : "left", - "primary_key" : [ "SITE_ID" ], - "foreign_key" : [ "LSTG_SITE_ID" ] - }, - "hierarchy" : null, - "table" : "TEST_SITES", - "column" : "{FK}", - "datatype" : "string", + "column" : ["TEST_SITES.SITE_ID"], "derived" : [ "SITE_NAME", "CRE_USER" ] }, { - "id" : 6, "name" : "SLR_SEGMENT_CD", - "join" : { - "type" : "left", - "primary_key" : [ "SELLER_TYPE_CD" ], - "foreign_key" : [ "SLR_SEGMENT_CD" ] - }, - "hierarchy" : null, - "table" : "TEST_SELLER_TYPE_DIM", - "column" : "{FK}", - "datatype" : "string", + "column" : ["TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD"], "derived" : [ "SELLER_TYPE_DESC" ] }, { - "id" : 7, "name" : "PRICE", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : "PRICE", - "datatype" : "decimal", + "column" : ["TEST_KYLIN_FACT.PRICE"], "derived" : null }, { - "id" : 8, "name" : "ITEM_COUNT", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : "ITEM_COUNT", - "datatype" : "bigint", + "column" : ["TEST_KYLIN_FACT.ITEM_COUNT"], "derived" : null }, { - "id" : 9, "name" : "SELLER_ID", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : "SELLER_ID", - "datatype" : "string", + "column" : ["TEST_KYLIN_FACT.SELLER_ID"], "derived" : null } ], "measures" : null, @@ -118,7 +57,6 @@ "signature" : null, "capacity" : "MEDIUM", "last_modified" : 1408328222841, - "fact_table" : "TEST_KYLIN_FACT", "filter_condition" : null, "cube_partition_desc" : { "partition_date_column" : null, diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json index 8bc0b17..148c315 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_desc.json @@ -1,7 +1,7 @@ { "uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92", "name": "test_kylin_cube_with_slr_desc", - "model_name": "test_kylin_with_slr_model_desc" + "model_name": "test_kylin_with_slr_model_desc", "filter_condition": null, "cube_partition_desc": { "partition_date_column": "test_kylin_fact.cal_dt", @@ -10,58 +10,33 @@ }, "dimensions": [ { - "id": "1", "name": "cal_dt", - "table": "test_cal_dt", - "column": "{FK}", + "column": ["test_cal_dt.cal_dt"], "derived": ["week_beg_dt"] }, { - "id": "2", "name": "category", - "table": "test_category_groupings", - "column": "{FK}", + "column": ["test_category_groupings.meta_categ_name", "test_category_groupings.categ_lvl2_name", "test_category_groupings.categ_lvl3_name"], "derived": ["USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER"], - "hierarchy": [ - { - "level": "1", - "column": "meta_categ_name" - }, - { - "level": "2", - "column": "categ_lvl2_name" - }, - { - "level": "3", - "column": "categ_lvl3_name" - } - ] + "hierarchy": true }, { - "id": "3", "name": "lstg_format_name", - "table": "test_kylin_fact", - "column": "lstg_format_name" + "column": ["test_kylin_fact.lstg_format_name"] }, { - "id": "4", "name": "site_id", - "table": "test_sites", - "column": "{FK}", + "column": ["test_sites.site_id"], "derived": ["site_name", "cre_user"] }, { - "id": "5", "name": "seller_type_cd", - "table": "test_seller_type_dim", - "column": "{FK}", + "column": ["test_seller_type_dim.slr_segment_cd"], "derived": ["seller_type_desc"] }, { - "id": "6", "name": "seller_id", - "table": "test_kylin_fact", - "column": "seller_id" + "column": ["test_kylin_fact.seller_id"] } ], "measures": [ diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json index 74f3996..2575f61 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json @@ -1,7 +1,7 @@ { "uuid": "bbbba905-1fc6-4f67-985c-38fa5aeafd92", "name": "test_kylin_cube_with_slr_left_join_desc", - "fact_table": "test_kylin_fact", + "model_name": "test_kylin_with_slr_left_join_model_desc", "cube_partition_desc": { "partition_date_column": "test_kylin_fact.cal_dt", "partition_date_start": 0, @@ -10,93 +10,37 @@ "filter_condition": null, "dimensions": [ { - "id": "1", "name": "cal_dt", - "datatype": "date", - "table": "test_cal_dt", - "column": "{FK}", - "derived": ["week_beg_dt"], - "join": { - "type": "left", - "primary_key": ["cal_dt"], - "foreign_key": ["cal_dt"] - } + "column": ["test_cal_dt.cal_dt"], + "derived": ["week_beg_dt"] }, { - "id": "2", "name": "category", - "table": "test_category_groupings", - "join": { - "type": "left", - "primary_key": ["leaf_categ_id", "site_id"], - "foreign_key": ["leaf_categ_id", "lstg_site_id"] - }, - "hierarchy": [ - { - "level": "1", - "column": "meta_categ_name" - }, - { - "level": "2", - "column": "categ_lvl2_name" - }, - { - "level": "3", - "column": "categ_lvl3_name" - } - ] + "column": ["test_category_groupings.meta_categ_name", "test_category_groupings.categ_lvl2_name", "test_category_groupings.categ_lvl3_name"], + "hierarchy": true }, { - "id": "3", "name": "category_derived", - "table": "test_category_groupings", - "column": "{FK}", - "derived": ["USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER"], - "join": { - "type": "left", - "primary_key": ["leaf_categ_id", "site_id"], - "foreign_key": ["leaf_categ_id", "lstg_site_id"] - } + "column": ["test_category_groupings.leaf_categ_id", "test_category_groupings.site_id"], + "derived": ["USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER"] }, { - "id": "4", "name": "lstg_format_name", - "datatype": "string", - "table": "test_kylin_fact", - "column": "lstg_format_name" + "column": ["test_kylin_fact.lstg_format_name"] }, { - "id": "5", "name": "site_id", - "datatype": "string", - "table": "test_sites", - "column": "{FK}", - "derived": ["site_name", "cre_user"], - "join": { - "type": "left", - "primary_key": ["site_id"], - "foreign_key": ["lstg_site_id"] - } + "column": ["test_sites.site_id"], + "derived": ["site_name", "cre_user"] }, { - "id": "6", "name": "seller_type_cd", - "datatype": "string", - "table": "test_seller_type_dim", - "column": "{FK}", - "derived": ["seller_type_desc"], - "join": { - "type": "left", - "primary_key": ["seller_type_cd"], - "foreign_key": ["slr_segment_cd"] - } + "column": ["test_seller_type_dim.seller_type_cd"], + "derived": ["seller_type_desc"] }, { - "id": "7", "name": "seller_id", - "datatype": "string", - "table": "test_kylin_fact", - "column": "seller_id" + "column": ["test_kylin_fact.seller_id"] } ], "measures": [ diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json index 4814020..547f4f6 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_desc.json @@ -1,8 +1,8 @@ { "uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf", "name": "test_kylin_cube_without_slr_desc", + "model_name": "test_kylin_without_slr_model_desc", "capacity": "SMALL", - "fact_table": "test_kylin_fact", "cube_partition_desc": { "partition_date_column": null, "partition_date_start": null, @@ -11,106 +11,42 @@ "filter_condition": null, "dimensions": [ { - "id": "1", "name": "cal_dt", - "datatype": "date", - "table": "test_cal_dt", - "column": "{FK}", + "column": ["test_cal_dt.cal_dt"], "derived": [ "week_beg_dt" - ], - "join": { - "type": "inner", - "primary_key": [ - "cal_dt" - ], - "foreign_key": [ - "cal_dt" - ] - } + ] }, { - "id": "2", "name": "category", "table": "test_category_groupings", - "column": "{FK}", + "column": ["test_category_groupings.meta_categ_name", "test_category_groupings.categ_lvl2_name", "test_category_groupings.categ_lvl3_name"], "derived": [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ], - "join": { - "type": "inner", - "primary_key": [ - "leaf_categ_id", - "site_id" - ], - "foreign_key": [ - "leaf_categ_id", - "lstg_site_id" - ] - }, - "hierarchy": [ - { - "level": "1", - "column": "meta_categ_name" - }, - { - "level": "2", - "column": "categ_lvl2_name" - }, - { - "level": "3", - "column": "categ_lvl3_name" - } - ] + "hierarchy": true }, { - "id": "3", "name": "lstg_format_name", - "datatype": "string", - "table": "test_kylin_fact", - "column": "lstg_format_name" + "column": ["test_kylin_fact.lstg_format_name"] }, { - "id": "4", "name": "site_id", - "datatype": "string", - "table": "test_sites", - "column": "{FK}", + "column": ["test_sites.site_id"], "derived": [ "site_name", "cre_user" - ], - "join": { - "type": "inner", - "primary_key": [ - "site_id" - ], - "foreign_key": [ - "lstg_site_id" - ] - } + ] }, { - "id": "5", "name": "seller_type_cd", - "datatype": "string", - "table": "test_seller_type_dim", - "column": "{FK}", + "column": ["test_seller_type_dim.seller_type_cd"], "derived": [ "seller_type_desc" - ], - "join": { - "type": "inner", - "primary_key": [ - "seller_type_cd" - ], - "foreign_key": [ - "slr_segment_cd" - ] - } + ] } ], "measures": [ diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json index d79ae72..15f6e78 100644 --- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json +++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json @@ -1,7 +1,7 @@ { "uuid": "9ac9b7a8-3929-4dff-b59d-2100aadc8dbf", "name": "test_kylin_cube_without_slr_left_join_desc", - "fact_table": "test_kylin_fact", + "model_name": "test_kylin_without_slr_left_join_model_desc", "cube_partition_desc": { "partition_date_column": "test_kylin_fact.cal_dt", "partition_date_start": 0, @@ -9,106 +9,41 @@ }, "dimensions": [ { - "id": "1", "name": "cal_dt", - "datatype": "date", - "table": "test_cal_dt", - "column": "{FK}", + "column": ["test_cal_dt.cal_dt"], "derived": [ "week_beg_dt" - ], - "join": { - "type": "left", - "primary_key": [ - "cal_dt" - ], - "foreign_key": [ - "cal_dt" - ] - } + ] }, { - "id": "2", "name": "category", - "table": "test_category_groupings", - "column": "{FK}", + "column": ["test_category_groupings.meta_categ_name", "test_category_groupings.categ_lvl2_name", "test_category_groupings.categ_lvl3_name"], "derived": [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ], - "join": { - "type": "left", - "primary_key": [ - "leaf_categ_id", - "site_id" - ], - "foreign_key": [ - "leaf_categ_id", - "lstg_site_id" - ] - }, - "hierarchy": [ - { - "level": "1", - "column": "meta_categ_name" - }, - { - "level": "2", - "column": "categ_lvl2_name" - }, - { - "level": "3", - "column": "categ_lvl3_name" - } - ] + "hierarchy": true }, { - "id": "3", "name": "lstg_format_name", - "datatype": "string", - "table": "test_kylin_fact", - "column": "lstg_format_name" + "column": ["test_kylin_fact.lstg_format_name"] }, { - "id": "4", "name": "site_id", - "datatype": "string", - "table": "test_sites", - "column": "{FK}", + "column": ["test_sites.site_id"], "derived": [ "site_name", "cre_user" - ], - "join": { - "type": "left", - "primary_key": [ - "site_id" - ], - "foreign_key": [ - "lstg_site_id" - ] - } + ] }, { - "id": "5", "name": "seller_type_cd", - "datatype": "string", - "table": "test_seller_type_dim", - "column": "{FK}", + "column": ["test_seller_type_dim.seller_type_cd"], "derived": [ "seller_type_desc" - ], - "join": { - "type": "left", - "primary_key": [ - "seller_type_cd" - ], - "foreign_key": [ - "slr_segment_cd" - ] - } + ] } ], "measures": [ diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json new file mode 100644 index 0000000..6c2d7b1 --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_ii_model_desc.json @@ -0,0 +1,38 @@ +{ + "name" : "test_kylin_ii_model_desc", + "fact_table": "TEST_KYLIN_FACT", + "lookups": [ + { + "table" : "TEST_CAL_DT", + "join" : { + "type" : "left", + "primary_key" : [ "CAL_DT" ], + "foreign_key" : [ "CAL_DT" ] + } + }, + { + "table" : "TEST_CATEGORY_GROUPINGS", + "join" : { + "type" : "left", + "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], + "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] + } + }, + { + "table" : "TEST_SITES", + "join" : { + "type" : "left", + "primary_key" : [ "SITE_ID" ], + "foreign_key" : [ "LSTG_SITE_ID" ] + } + }, + { + "table" : "TEST_SELLER_TYPE_DIM", + "join" : { + "type" : "left", + "primary_key" : [ "SELLER_TYPE_CD" ], + "foreign_key" : [ "SLR_SEGMENT_CD" ] + } + } +] +} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json new file mode 100644 index 0000000..acb8e0c --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_left_join_model_desc.json @@ -0,0 +1,46 @@ +{ + "name": "test_kylin_with_slr_left_join_model_desc", + "fact_table": "test_kylin_fact", + "lookups": [ + { + "table": "test_cal_dt", + "join": { + "type": "left", + "primary_key": ["cal_dt"], + "foreign_key": ["cal_dt"] + } + }, + { + "table": "test_category_groupings", + "join": { + "type": "left", + "primary_key": ["leaf_categ_id", "site_id"], + "foreign_key": ["leaf_categ_id", "lstg_site_id"] + } + }, + { + "table": "test_category_groupings", + "join": { + "type": "left", + "primary_key": ["leaf_categ_id", "site_id"], + "foreign_key": ["leaf_categ_id", "lstg_site_id"] + } + }, + { + "table": "test_sites", + "join": { + "type": "left", + "primary_key": ["site_id"], + "foreign_key": ["lstg_site_id"] + } + }, + { + "table": "test_seller_type_dim", + "join": { + "type": "left", + "primary_key": ["seller_type_cd"], + "foreign_key": ["slr_segment_cd"] + } + } + ] +} diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json index 7209e88..b1347a1 100644 --- a/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_with_slr_model_desc.json @@ -1,10 +1,8 @@ { - "uuid": "a24ca905-1fc6-4f67-985c-341fewaeafd92", "name": "test_kylin_with_slr_model_desc", "fact_table": "test_kylin_fact", "lookups": [ { - "id": "1", "table": "test_cal_dt", "join": { "type": "inner", @@ -13,7 +11,6 @@ } }, { - "id": "2", "table": "test_category_groupings", "join": { "type": "inner", @@ -22,7 +19,6 @@ } }, { - "id": "3", "table": "test_sites", "join": { "type": "inner", @@ -31,7 +27,6 @@ } }, { - "id": "4", "table": "test_seller_type_dim", "join": { "type": "inner", diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json new file mode 100644 index 0000000..5773190 --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_left_join_model_desc.json @@ -0,0 +1,56 @@ +{ + "name": "test_kylin_without_slr_left_join_model_desc", + "fact_table": "test_kylin_fact", + "lookups": [ + { + "table": "test_cal_dt", + "join": { + "type": "left", + "primary_key": [ + "cal_dt" + ], + "foreign_key": [ + "cal_dt" + ] + } + }, + { + "table": "test_category_groupings", + "join": { + "type": "left", + "primary_key": [ + "leaf_categ_id", + "site_id" + ], + "foreign_key": [ + "leaf_categ_id", + "lstg_site_id" + ] + } + }, + { + "table": "test_sites", + "join": { + "type": "left", + "primary_key": [ + "site_id" + ], + "foreign_key": [ + "lstg_site_id" + ] + } + }, + { + "table": "test_seller_type_dim", + "join": { + "type": "left", + "primary_key": [ + "seller_type_cd" + ], + "foreign_key": [ + "slr_segment_cd" + ] + } + } + ] +} \ No newline at end of file diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json new file mode 100644 index 0000000..09319ab --- /dev/null +++ b/examples/test_case_data/localmeta/model_desc/test_kylin_without_slr_model_desc.json @@ -0,0 +1,56 @@ +{ + "name": "test_kylin_without_slr_model_desc", + "fact_table": "test_kylin_fact", + "lookups": [ + { + "table" : "test_cal_dt", + "join": { + "type": "inner", + "primary_key": [ + "cal_dt" + ], + "foreign_key": [ + "cal_dt" + ] + } + }, + { + "table": "test_category_groupings", + "join": { + "type": "inner", + "primary_key": [ + "leaf_categ_id", + "site_id" + ], + "foreign_key": [ + "leaf_categ_id", + "lstg_site_id" + ] + } + }, + { + "table": "test_sites", + "join": { + "type": "inner", + "primary_key": [ + "site_id" + ], + "foreign_key": [ + "lstg_site_id" + ] + } + }, + { + "table": "test_seller_type_dim", + "join": { + "type": "inner", + "primary_key": [ + "seller_type_cd" + ], + "foreign_key": [ + "slr_segment_cd" + ] + } + } + ] +} \ No newline at end of file diff --git a/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java b/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java index 5aaf67e..31a0b2e 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java +++ b/metadata/src/main/java/com/kylinolap/metadata/MetadataManager.java @@ -25,7 +25,6 @@ import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; -import com.kylinolap.metadata.model.DataModelDesc; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,14 +38,15 @@ import com.kylinolap.common.restclient.Broadcaster; import com.kylinolap.common.restclient.SingleValueCache; import com.kylinolap.common.util.JsonUtil; +import com.kylinolap.metadata.model.DataModelDesc; import com.kylinolap.metadata.model.TableDesc; import com.kylinolap.metadata.model.invertedindex.InvertedIndexDesc; /** - * Serves (and caches) cube metadata for Kylin instance. + * Serves (and caches) metadata for Kylin instance. *

- * Also provides a ResourceStore for general purpose data persistence. Cube - * metadata is serialized as JSON and stored in ResourceStore. + * Also provides a ResourceStore for general purpose data persistence. + * Metadata is serialized as JSON and stored in ResourceStore. * * @author yangli9 */ @@ -56,6 +56,7 @@ private static final Serializer TABLE_SERIALIZER = new JsonSerializer(TableDesc.class); private static final Serializer IIDESC_SERIALIZER = new JsonSerializer(InvertedIndexDesc.class); + private static final Serializer MODELDESC_SERIALIZER = new JsonSerializer(DataModelDesc.class); TypeReference> typeRef = new TypeReference>() { }; @@ -83,7 +84,7 @@ public static MetadataManager getInstance(KylinConfig config) { return r; } catch (IOException e) { - throw new IllegalStateException("Failed to init CubeManager from " + config, e); + throw new IllegalStateException("Failed to init MetadataManager from " + config, e); } } } @@ -105,6 +106,8 @@ public static void dropCache() { private SingleValueCache iiDescMap = new SingleValueCache(Broadcaster.TYPE.METADATA); // name => value private SingleValueCache> srcTableExdMap = new SingleValueCache>(Broadcaster.TYPE.METADATA); + // name => DataModelDesc + private SingleValueCache dataModelDescMap = new SingleValueCache(Broadcaster.TYPE.METADATA); private MetadataManager(KylinConfig config) throws IOException { init(config); @@ -186,6 +189,7 @@ private void init(KylinConfig config) throws IOException { reloadAllSourceTable(); reloadAllSourceTableExd(); reloadAllInvertedIndexDesc(); + reloadAllDataModel(); } private void reloadAllSourceTableExd() throws IOException { @@ -300,7 +304,7 @@ private InvertedIndexDesc loadInvertedIndexDesc(String path) throws IOException } /** - * Tell CubeManager that the cube instance has changed. The cube info will + * Tell MetadataManager that the instance has changed. The cube info will * be stored Reload the cube desc and source table A broadcast must be sent * out * @@ -313,19 +317,112 @@ public void reload() { } public DataModelDesc getDataModelDesc(String name) { - return null; + return dataModelDescMap.get(name); } + + + private void reloadAllDataModel() throws IOException { + ResourceStore store = getStore(); + logger.debug("Reloading DataModel from folder " + store.getReadableResourcePath(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT)); - public void updateDataModelDesc(DataModelDesc dataModelDesc) { - throw new UnsupportedOperationException(); + this.dataModelDescMap.clear(); + + List paths = store.collectResourceRecursively(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT, MetadataConstances.FILE_SURFIX); + for (String path : paths) { + DataModelDesc modelDesc = this.loadDataModelDesc(path); + dataModelDescMap.putLocal(modelDesc.getName(), modelDesc); + } + + logger.debug("Loaded " + paths.size() + " DataModel(s)"); } - public DataModelDesc createDataModelDesc(DataModelDesc dataModelDesc) { + public DataModelDesc createDataModelDesc(DataModelDesc dataModelDesc) throws IOException { + if (dataModelDescMap.containsKey(dataModelDesc.getName())) + throw new IllegalArgumentException("DataModelDesc '" + dataModelDesc.getName() + "' already exists"); + + try { + dataModelDesc.init(this.getAllTablesMap()); + } catch (IllegalStateException e) { + dataModelDesc.addError(e.getMessage(), true); + } + // Check base validation + if (!dataModelDesc.getError().isEmpty()) { + return dataModelDesc; + } + + String path = dataModelDesc.getResourcePath(); + getStore().putResource(path, dataModelDesc, MODELDESC_SERIALIZER); + dataModelDescMap.put(dataModelDesc.getName(), dataModelDesc); + return dataModelDesc; } + + /** + * Update DataModelDesc with the input. Broadcast the event into cluster + * + * @param desc + * @return + * @throws IOException + */ + public DataModelDesc updateDataModelDesc(DataModelDesc desc) throws IOException { + String name = desc.getName(); + if (!dataModelDescMap.containsKey(name)) { + throw new IllegalArgumentException("DataModelDesc '" + name + "' does not exist."); + } + + try { + desc.init(this.getAllTablesMap()); + } catch (IllegalStateException e) { + desc.addError(e.getMessage(), true); + return desc; + } catch (IllegalArgumentException e) { + desc.addError(e.getMessage(), true); + return desc; + } + + + // Save Source + String path = desc.getResourcePath(); + getStore().putResource(path, desc, MODELDESC_SERIALIZER); + + // Reload the DataModelDesc + DataModelDesc ndesc = loadDataModelDesc(path); + // Here replace the old one + dataModelDescMap.put(ndesc.getName(), desc); + + return ndesc; + } + + private DataModelDesc loadDataModelDesc(String path) throws IOException { + ResourceStore store = getStore(); + logger.debug("Loading DataModelDesc " + store.getReadableResourcePath(path)); + DataModelDesc ndesc = null; + try { + ndesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER); + + } catch (IOException e) { + System.err.println("Error to load" + path + ", exception is " + e.toString()); + throw e; + } + if (StringUtils.isBlank(ndesc.getName())) { + throw new IllegalStateException("DataModelDesc name must not be blank"); + } + + ndesc.init(this.getAllTablesMap()); + + if (ndesc.getError().isEmpty() == false) { + throw new IllegalStateException("DataModelDesc at " + path + " has issues: " + ndesc.getError()); + } + + return ndesc; + } + - public boolean deleteDataModelDesc(DataModelDesc dataModelDesc) { - throw new UnsupportedOperationException(); + public void deleteDataModelDesc(DataModelDesc dataModelDesc) throws IOException { + // remove dataModelDesc + String path = dataModelDesc.getResourcePath(); + getStore().deleteResource(path); + dataModelDescMap.remove(dataModelDesc.getName()); } } diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java index 3e95ee9..b0f06ad 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java +++ b/metadata/src/main/java/com/kylinolap/metadata/model/DataModelDesc.java @@ -1,24 +1,38 @@ package com.kylinolap.metadata.model; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + import org.apache.commons.lang.ArrayUtils; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import com.kylinolap.common.persistence.ResourceStore; +import com.kylinolap.common.persistence.RootPersistentEntity; +import com.kylinolap.common.util.StringUtil; +import com.kylinolap.metadata.MetadataConstances; import com.kylinolap.metadata.model.realization.TblColRef; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class DataModelDesc { - +public class DataModelDesc extends RootPersistentEntity { + @JsonProperty("name") private String name; - + @JsonProperty("fact_table") private String factTable; @JsonProperty("lookups") private LookupDesc[] lookups; + /** + * Error messages during resolving json metadata + */ + private List errors = new ArrayList(); + public String getName() { return name; } @@ -42,11 +56,11 @@ public void setFactTable(String factTable) { public void setLookups(LookupDesc[] lookups) { this.lookups = lookups; } - + public boolean isFactTable(String factTable) { return this.factTable.equalsIgnoreCase(factTable); } - + public TblColRef findPKByFK(TblColRef fk) { assert isFactTable(fk.getTable()); @@ -68,5 +82,92 @@ public TblColRef findPKByFK(TblColRef fk) { } return candidate; } - + + public void init(Map tables) { + this.errors.clear(); + this.factTable = factTable.toUpperCase(); + initJoinColumns(tables); + } + + private void initJoinColumns(Map tables) { + // join columns may or may not present in cube; + // here we don't modify 'allColumns' and 'dimensionColumns'; + // initDimensionColumns() will do the update + for (LookupDesc lookup : this.lookups) { + lookup.setTable(lookup.getTable().toUpperCase()); + TableDesc dimTable = tables.get(lookup.getTable()); + + JoinDesc join = lookup.getJoin(); + if (join == null) + continue; + + StringUtil.toUpperCaseArray(join.getForeignKey(), join.getForeignKey()); + StringUtil.toUpperCaseArray(join.getPrimaryKey(), join.getPrimaryKey()); + // primary key + String[] pks = join.getPrimaryKey(); + TblColRef[] pkCols = new TblColRef[pks.length]; + for (int i = 0; i < pks.length; i++) { + ColumnDesc col = dimTable.findColumnByName(pks[i]); + if (col == null) { + addError("Can't find column " + pks[i] + " in table " + dimTable.getName()); + } + TblColRef colRef = new TblColRef(col); + pks[i] = colRef.getName(); + pkCols[i] = colRef; + } + join.setPrimaryKeyColumns(pkCols); + // foreign key + TableDesc factTable = tables.get(this.getFactTable()); + if (factTable == null) { + addError("Fact table does not exist:" + this.getFactTable()); + } + String[] fks = join.getForeignKey(); + TblColRef[] fkCols = new TblColRef[fks.length]; + for (int i = 0; i < fks.length; i++) { + ColumnDesc col = factTable.findColumnByName(fks[i]); + if (col == null) { + addError("Can't find column " + fks[i] + " in table " + this.getFactTable()); + } + TblColRef colRef = new TblColRef(col); + fks[i] = colRef.getName(); + fkCols[i] = colRef; + } + join.setForeignKeyColumns(fkCols); + // Validate join in dimension + if (pkCols.length != fkCols.length) { + addError("Primary keys(" + lookup.getTable() + ")" + Arrays.toString(pks) + " are not consistent with Foreign keys(" + this.getFactTable() + ") " + Arrays.toString(fks)); + } + for (int i = 0; i < fkCols.length; i++) { + if (!fkCols[i].getDatatype().equals(pkCols[i].getDatatype())) { + addError("Primary key " + lookup.getTable() + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with Foreign key " + this.getFactTable() + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype()); + } + } + + } + } + + public String getResourcePath() { + return getDataModelDescResourcePath(name); + } + + public static String getDataModelDescResourcePath(String descName) { + return ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstances.FILE_SURFIX; + } + + public void addError(String message) { + addError(message, false); + } + + public void addError(String message, boolean silent) { + if (!silent) { + throw new IllegalStateException(message); + } else { + this.errors.add(message); + } + } + + public List getError() { + return this.errors; + } + } diff --git a/metadata/src/test/resources/data/TEST1_desc.json b/metadata/src/test/resources/data/TEST1_desc.json deleted file mode 100644 index 175b105..0000000 --- a/metadata/src/test/resources/data/TEST1_desc.json +++ /dev/null @@ -1,192 +0,0 @@ -{ - "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b", - "last_modified" : 1401429176099, - "name" : "TEST1_desc", - "fact_table" : "TEST_KYLIN_FACT", - "dimensions" : [ { - "id" : 1, - "name" : "CAL_DT", - "join" : { - "type" : "inner", - "primary_key" : [ "CAL_DT" ], - "foreign_key" : [ "CAL_DT" ] - }, - "hierarchy" : null, - "table" : "TEST_CAL_DT", - "column" : "CAL_DT", - "datatype" : "date", - "derived" : [ "WEEK_BEG_DT" ] - }, { - "id" : 2, - "name" : "CATEGORY", - "join" : { - "type" : "inner", - "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], - "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] - }, - "hierarchy" : [ { - "level" : "1", - "column" : "META_CATEG_NAME" - }, { - "level" : "2", - "column" : "CATEG_LVL2_NAME" - }, { - "level" : "3", - "column" : "CATEG_LVL3_NAME" - } ], - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : null, - "datatype" : null, - "derived" : null - }, { - "id" : 3, - "name" : "LSTG_FORMAT_NAME", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : "LSTG_FORMAT_NAME", - "datatype" : "string", - "derived" : null - }, { - "id" : 4, - "name" : "SITE_ID", - "join" : { - "type" : "inner", - "primary_key" : [ "SITE_ID" ], - "foreign_key" : [ "LSTG_SITE_ID" ] - }, - "hierarchy" : null, - "table" : "TEST_SITES", - "column" : "SITE_ID", - "datatype" : "string", - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "id" : 5, - "name" : "SELLER_TYPE_CD", - "join" : { - "type" : "inner", - "primary_key" : [ "SELLER_TYPE_CD" ], - "foreign_key" : [ "SLR_SEGMENT_CD" ] - }, - "hierarchy" : null, - "table" : "TEST_SELLER_TYPE_DIM", - "column" : "SELLER_TYPE_CD", - "datatype" : "string", - "derived" : [ "SELLER_TYPE_DESC" ] - } ], - "measures" : [ { - "id" : 1, - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 2, - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 3, - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 4, - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1" - }, - "returntype" : "long" - } - }, { - "id" : 5, - "name" : "SELLER_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "SELLER_ID" - }, - "returntype" : "hllc10" - } - }, { - "id" : 6, - "name" : "SELLER_FORMAT_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "LSTG_FORMAT_NAME,SELLER_ID" - }, - "returntype" : "hllc10" - } - } ], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "CAL_DT", - "length" : 0, - "dictionary" : "date(yyyy-mm-dd)", - "mandatory" : true - }, { - "column" : "META_CATEG_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL2_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL3_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "LSTG_FORMAT_NAME", - "length" : 12, - "dictionary" : null, - "mandatory" : false - }, { - "column" : "SITE_ID", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "SELLER_TYPE_CD", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - } ], - "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CATEG_LVL2_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD" ] ] - }, - "hbase_mapping" : { - "column_family" : [ { - "name" : "F1", - "columns" : [ { - "qualifier" : "M", - "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ] - } ] - } ] - } -} \ No newline at end of file diff --git a/metadata/src/test/resources/data/TEST2_desc.json b/metadata/src/test/resources/data/TEST2_desc.json deleted file mode 100644 index 81bde27..0000000 --- a/metadata/src/test/resources/data/TEST2_desc.json +++ /dev/null @@ -1,192 +0,0 @@ -{ - "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b", - "last_modified" : 1401429176099, - "name" : "TEST2_desc", - "fact_table" : "TEST_KYLIN_FACT", - "dimensions" : [ { - "id" : 1, - "name" : "CAL_DT", - "join" : { - "type" : "inner", - "primary_key" : [ "CAL_DT" ], - "foreign_key" : [ "CAL_DT" ] - }, - "hierarchy" : null, - "table" : "TEST_CAL_DT", - "column" : "CAL_DT", - "datatype" : "date", - "derived" : [ "WEEK_BEG_DT" ] - }, { - "id" : 2, - "name" : "CATEGORY", - "join" : { - "type" : "inner", - "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], - "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] - }, - "hierarchy" : [ { - "level" : "1", - "column" : "META_CATEG_NAME" - }, { - "level" : "2", - "column" : "CATEG_LVL2_NAME" - }, { - "level" : "3", - "column" : "CATEG_LVL3_NAME" - } ], - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : null, - "datatype" : null, - "derived" : null - }, { - "id" : 3, - "name" : "LSTG_FORMAT_NAME", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : "LSTG_FORMAT_NAME", - "datatype" : "string", - "derived" : null - }, { - "id" : 4, - "name" : "SITE_ID", - "join" : { - "type" : "inner", - "primary_key" : [ "SITE_ID" ], - "foreign_key" : [ "LSTG_SITE_ID" ] - }, - "hierarchy" : null, - "table" : "TEST_SITES", - "column" : "SITE_ID", - "datatype" : "string", - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "id" : 5, - "name" : "SELLER_TYPE_CD", - "join" : { - "type" : "inner", - "primary_key" : [ "SELLER_TYPE_CD" ], - "foreign_key" : [ "SLR_SEGMENT_CD" ] - }, - "hierarchy" : null, - "table" : "TEST_SELLER_TYPE_DIM", - "column" : "SELLER_TYPE_CD", - "datatype" : "string", - "derived" : [ "SELLER_TYPE_DESC" ] - } ], - "measures" : [ { - "id" : 1, - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 2, - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 3, - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 4, - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1" - }, - "returntype" : "long" - } - }, { - "id" : 5, - "name" : "SELLER_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "SELLER_ID" - }, - "returntype" : "hllc10" - } - }, { - "id" : 6, - "name" : "SELLER_FORMAT_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "LSTG_FORMAT_NAME,SELLER_ID" - }, - "returntype" : "hllc10" - } - } ], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "CAL_DT", - "length" : 0, - "dictionary" : "date(yyyy-mm-dd)", - "mandatory" : true - }, { - "column" : "META_CATEG_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL2_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL3_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "LSTG_FORMAT_NAME", - "length" : 12, - "dictionary" : null, - "mandatory" : false - }, { - "column" : "SITE_ID", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "SELLER_TYPE_CD", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - } ], - "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD", "CATEG_LVL2_NAME" ] ] - }, - "hbase_mapping" : { - "column_family" : [ { - "name" : "F1", - "columns" : [ { - "qualifier" : "M", - "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ] - } ] - } ] - } -} \ No newline at end of file diff --git a/metadata/src/test/resources/data/TEST3_desc.json b/metadata/src/test/resources/data/TEST3_desc.json deleted file mode 100644 index df4d60b..0000000 --- a/metadata/src/test/resources/data/TEST3_desc.json +++ /dev/null @@ -1,192 +0,0 @@ -{ - "uuid" : "9c53506d-d7b9-4ad4-b3b1-53ea42673c4b", - "last_modified" : 1401429176099, - "name" : "TEST1_desc", - "fact_table" : "TEST_KYLIN_FACT", - "dimensions" : [ { - "id" : 1, - "name" : "CAL_DT", - "join" : { - "type" : "inner", - "primary_key" : [ "CAL_DT" ], - "foreign_key" : [ "CAL_DT" ] - }, - "hierarchy" : null, - "table" : "TEST_CAL_DT", - "column" : "CAL_DT", - "datatype" : "date", - "derived" : [ "WEEK_BEG_DT" ] - }, { - "id" : 2, - "name" : "CATEGORY", - "join" : { - "type" : "inner", - "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ], - "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ] - }, - "hierarchy" : [ { - "level" : "1", - "column" : "META_CATEG_NAME" - }, { - "level" : "2", - "column" : "CATEG_LVL2_NAME" - }, { - "level" : "3", - "column" : "CATEG_LVL3_NAME" - } ], - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : null, - "datatype" : null, - "derived" : null - }, { - "id" : 3, - "name" : "LSTG_FORMAT_NAME", - "join" : null, - "hierarchy" : null, - "table" : "TEST_KYLIN_FACT", - "column" : "LSTG_FORMAT_NAME", - "datatype" : "string", - "derived" : null - }, { - "id" : 4, - "name" : "SITE_ID", - "join" : { - "type" : "inner", - "primary_key" : [ "SITE_ID" ], - "foreign_key" : [ "LSTG_SITE_ID" ] - }, - "hierarchy" : null, - "table" : "TEST_SITES", - "column" : "SITE_ID", - "datatype" : "string", - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "id" : 5, - "name" : "SELLER_TYPE_CD", - "join" : { - "type" : "inner", - "primary_key" : [ "SELLER_TYPE_CD" ], - "foreign_key" : [ "SLR_SEGMENT_CD" ] - }, - "hierarchy" : null, - "table" : "TEST_SELLER_TYPE_DIM", - "column" : "SELLER_TYPE_CD", - "datatype" : "string", - "derived" : [ "SELLER_TYPE_DESC" ] - } ], - "measures" : [ { - "id" : 1, - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 2, - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 3, - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "PRICE" - }, - "returntype" : "decimal" - } - }, { - "id" : 4, - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1" - }, - "returntype" : "long" - } - }, { - "id" : 5, - "name" : "SELLER_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "SELLER_ID" - }, - "returntype" : "hllc10" - } - }, { - "id" : 6, - "name" : "SELLER_FORMAT_CNT", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "LSTG_FORMAT_NAME,SELLER_ID" - }, - "returntype" : "hllc10" - } - } ], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "CAL_DT", - "length" : 10, - "dictionary" : "date(yyyy-mm-dd)", - "mandatory" : false - }, { - "column" : "META_CATEG_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL2_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "CATEG_LVL3_NAME", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "LSTG_FORMAT_NAME", - "length" : 12, - "dictionary" : null, - "mandatory" : false - }, { - "column" : "SITE_ID", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - }, { - "column" : "SELLER_TYPE_CD", - "length" : 0, - "dictionary" : "string", - "mandatory" : false - } ], - "aggregation_groups" : [ [ "META_CATEG_NAME", "CATEG_LVL3_NAME", "CATEG_LVL2_NAME", "CAL_DT" ], [ "LSTG_FORMAT_NAME", "SITE_ID", "SELLER_TYPE_CD" ] ] - }, - "hbase_mapping" : { - "column_family" : [ { - "name" : "F1", - "columns" : [ { - "qualifier" : "M", - "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "SELLER_CNT", "SELLER_FORMAT_CNT" ] - } ] - } ] - } -} \ No newline at end of file diff --git a/storage/src/test/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpoindAggregationTest.java b/storage/src/test/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpoindAggregationTest.java index 5bd0d1d..04980c3 100644 --- a/storage/src/test/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpoindAggregationTest.java +++ b/storage/src/test/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpoindAggregationTest.java @@ -2,6 +2,18 @@ import static org.junit.Assert.*; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + import com.kylinolap.common.util.BytesUtil; import com.kylinolap.common.util.LocalFileMetadataTestCase; import com.kylinolap.cube.CubeInstance; @@ -17,16 +29,11 @@ import com.kylinolap.storage.filter.TupleFilter; import com.kylinolap.storage.hbase.coprocessor.CoprocessorFilter; import com.kylinolap.storage.hbase.coprocessor.CoprocessorProjector; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.*; /** * Created by Hongbin Ma(Binmahone) on 11/27/14. */ +@Ignore public class EndpoindAggregationTest extends LocalFileMetadataTestCase { CubeInstance cube; TableRecordInfo tableRecordInfo;