From 596d5bed43eae35fd4fc74cfa8d7d5ad79c00305 Mon Sep 17 00:00:00 2001 From: Albert Lee Date: Sat, 9 Sep 2017 14:52:32 +0800 Subject: [PATCH 1/3] Add some function to support cube table can be created in specific configuration. --- .../org/apache/kylin/common/KylinConfigBase.java | 22 ++++++++++++++++++++++ .../java/org/apache/kylin/cube/CubeManager.java | 10 ++++++---- .../org/apache/kylin/cube/CubeManagerTest.java | 10 +++++++++- .../realization/IRealizationConstants.java | 1 - examples/test_case_data/localmeta/kylin.properties | 6 ++++++ examples/test_case_data/sandbox/kylin.properties | 6 ++++++ 6 files changed, 49 insertions(+), 6 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index f9c3adb..141132c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -382,6 +382,28 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.cube.gtscanrequest-serialization-level", "1")); } + public String getCubePrefixOfNamespace() { + String namespace = getOptional("kylin.storage.location.prefix.hnamespace"); + if(namespace != null || namespace.equals("")) { + return namespace + ":"; + } else { + return ""; + } + } + + public String getCubePrefixOfHTable() { + String tableprefix = getOptional("kylin.storage.location.prefix.htable"); + if(tableprefix != null || tableprefix.equals("")) { + if(tableprefix.endsWith("_")) { + return tableprefix; + } else { + return tableprefix + "_"; + } + } else { + return "KYLIN_"; + } + } + // ============================================================================ // JOB // ============================================================================ diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 0beb926..3e4abb2 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -35,6 +35,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinConfigExt; @@ -62,7 +63,6 @@ import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.IRealizationConstants; import org.apache.kylin.metadata.realization.IRealizationProvider; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.metadata.realization.RealizationType; @@ -708,8 +708,10 @@ public class CubeManager implements IRealizationProvider { return segment; } - private String generateStorageLocation() { - String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix; + @VisibleForTesting + /*private*/ String generateStorageLocation() { + KylinConfig config = KylinConfig.getInstanceFromEnv(); + String namePrefix = config.getCubePrefixOfNamespace() + config.getCubePrefixOfHTable(); String tableName = ""; Random ran = new Random(); do { @@ -950,7 +952,7 @@ public class CubeManager implements IRealizationProvider { private final String GLOBAL_DICTIONNARY_CLASS = "org.apache.kylin.dict.GlobalDictionaryBuilder"; - //UHC (ultra high cardinality column): contain the ShardByColumns and the GlobalDictionaryColumns + //UHC (ultra high cardinality column): contain the Shardopen ByColumns and the GlobalDictionaryColumns public int[] getUHCIndex(CubeDesc cubeDesc) throws IOException { List factDictCols = getAllDictColumnsOnFact(cubeDesc); int[] uhcIndex = new int[factDictCols.size()]; diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index 3cae37d..e826cda 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -48,7 +48,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { - this.createTestMetadata(); + staticCreateTestMetadata(); } @After @@ -304,6 +304,14 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { assertTrue(mergedSeg.getFirst() == 0 && mergedSeg.getSecond() == 8000); } + @Test + public void testGetCubeNameWithNamespace() { + CubeManager mgr = CubeManager.getInstance(getTestConfig()); + String tablename = mgr.generateStorageLocation(); + System.out.println(tablename); + assertTrue(tablename.startsWith("HELLO:WORLD")); + } + public CubeDescManager getCubeDescManager() { return CubeDescManager.getInstance(getTestConfig()); } diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java index 2a3b01e..e232b0f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java @@ -23,7 +23,6 @@ package org.apache.kylin.metadata.realization; public class IRealizationConstants { public final static String SharedHbaseStorageLocationPrefix = "KYLIN_"; - public final static String CubeHbaseStorageLocationPrefix = "KYLIN_"; public final static String IIHbaseStorageLocationPrefix = "KYLIN_II_"; /** diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index c7dda3f..0600751 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -33,6 +33,12 @@ kylin.storage.hbase.owner-tag=whoami@kylin.apache.org ### STORAGE ### +# The namespace for hbase. If don't set, hbase will use "default" namespace. +kylin.storage.location.prefix.hnamespace=HELLO + +# The tablename prefix for hbase. If don't set, hbase will use "KYLIN_" as cube tablename prefix. +kylin.storage.location.prefix.htable=WORLD + # The metadata store in hbase kylin.metadata.url= diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 6a571df..60a2ac9 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -38,6 +38,12 @@ kylin.source.hive.client=cli ### STORAGE ### +# The namespace for hbase. If don't set, hbase will use "default" namespace. +kylin.storage.location.prefix.hnamespace= + +# The tablename prefix for hbase. If don't set, hbase will use "KYLIN_" as cube tablename prefix. +kylin.storage.location.prefix.htable= + # The metadata store in hbase kylin.metadata.url=kylin_metadata@hbase -- 2.4.9 (Apple Git-60) From f7fc8e8d6230534fc972fbd93f9f5adaf478de96 Mon Sep 17 00:00:00 2001 From: Albert Lee Date: Sat, 9 Sep 2017 16:04:42 +0800 Subject: [PATCH 2/3] Add some function to support cube table can be created in specific configuration. --- .../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++-- core-common/src/main/resources/kylin-defaults.properties | 6 ++++++ examples/test_case_data/localmeta/kylin.properties | 12 ++++++------ examples/test_case_data/sandbox/kylin.properties | 13 ++++++------- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 141132c..1ef8862 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -384,7 +384,7 @@ abstract public class KylinConfigBase implements Serializable { public String getCubePrefixOfNamespace() { String namespace = getOptional("kylin.storage.location.prefix.hnamespace"); - if(namespace != null || namespace.equals("")) { + if((namespace != null) && (!namespace.equals(""))) { return namespace + ":"; } else { return ""; @@ -393,7 +393,7 @@ abstract public class KylinConfigBase implements Serializable { public String getCubePrefixOfHTable() { String tableprefix = getOptional("kylin.storage.location.prefix.htable"); - if(tableprefix != null || tableprefix.equals("")) { + if((tableprefix != null) && (!tableprefix.equals(""))) { if(tableprefix.endsWith("_")) { return tableprefix; } else { diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties index cb511e7..bab72aa 100644 --- a/core-common/src/main/resources/kylin-defaults.properties +++ b/core-common/src/main/resources/kylin-defaults.properties @@ -63,6 +63,12 @@ kylin.source.hive.redistribute-flat-table=true # The storage for final cube file in hbase kylin.storage.url=hbase +# The namespace for hbase table. If don't set, hbase will use "default" namespace. +kylin.storage.location.prefix.hnamespace= + +# The hbase tablename prefix. If don't set, hbase will use "KYLIN_". +kylin.storage.location.prefix.htable= + # Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4] kylin.storage.hbase.compression-codec=none diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index 0600751..5d38f59 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -33,18 +33,18 @@ kylin.storage.hbase.owner-tag=whoami@kylin.apache.org ### STORAGE ### -# The namespace for hbase. If don't set, hbase will use "default" namespace. -kylin.storage.location.prefix.hnamespace=HELLO - -# The tablename prefix for hbase. If don't set, hbase will use "KYLIN_" as cube tablename prefix. -kylin.storage.location.prefix.htable=WORLD - # The metadata store in hbase kylin.metadata.url= # The storage for final cube file in hbase kylin.storage.url=hbase +# The namespace for hbase table. If don't set, hbase will use "default" namespace. +kylin.storage.location.prefix.hnamespace=HELLO + +# The cube hbase tablename prefix. If don't set, hbase will use "KYLIN_". +kylin.storage.location.prefix.htable=WORLD + # Working folder in HDFS, make sure user has the right access to the hdfs directory kylin.env.hdfs-working-dir=/kylin diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 60a2ac9..49bbf86 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -38,19 +38,18 @@ kylin.source.hive.client=cli ### STORAGE ### -# The namespace for hbase. If don't set, hbase will use "default" namespace. -kylin.storage.location.prefix.hnamespace= - -# The tablename prefix for hbase. If don't set, hbase will use "KYLIN_" as cube tablename prefix. -kylin.storage.location.prefix.htable= - # The metadata store in hbase kylin.metadata.url=kylin_metadata@hbase - # The storage for final cube file in hbase kylin.storage.url=hbase +# The namespace for hbase table. If don't set, hbase will use "default" namespace. +kylin.storage.location.prefix.hnamespace= + +# The hbase tablename prefix. If don't set, hbase will use "KYLIN_". +kylin.storage.location.prefix.htable=KYLIN + # Working folder in HDFS, make sure user has the right access to the hdfs directory kylin.env.hdfs-working-dir=/kylin -- 2.4.9 (Apple Git-60) From ce54928631af246bf2a79e7c4b7ccad7fcd08bc4 Mon Sep 17 00:00:00 2001 From: Albert Lee Date: Sat, 9 Sep 2017 16:35:20 +0800 Subject: [PATCH 3/3] Add some function to support cube table can be created in specific configuration. --- core-common/src/main/resources/kylin-defaults.properties | 4 ++-- core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties index bab72aa..3c07c01 100644 --- a/core-common/src/main/resources/kylin-defaults.properties +++ b/core-common/src/main/resources/kylin-defaults.properties @@ -64,10 +64,10 @@ kylin.source.hive.redistribute-flat-table=true kylin.storage.url=hbase # The namespace for hbase table. If don't set, hbase will use "default" namespace. -kylin.storage.location.prefix.hnamespace= +#kylin.storage.location.prefix.hnamespace= # The hbase tablename prefix. If don't set, hbase will use "KYLIN_". -kylin.storage.location.prefix.htable= +#kylin.storage.location.prefix.htable= # Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4] kylin.storage.hbase.compression-codec=none diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index e826cda..d3276ad 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -48,7 +48,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { @Before public void setUp() throws Exception { - staticCreateTestMetadata(); + this.createTestMetadata(); } @After -- 2.4.9 (Apple Git-60)