From 6c36f6bd44de6cc985714b743a10b2210f83efb7 Mon Sep 17 00:00:00 2001 From: Albert Lee Date: Sun, 10 Sep 2017 23:23:13 +0800 Subject: [PATCH 1/3] Add function for cube hbase tablename prefix. --- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++ core-common/src/main/resources/kylin-defaults.properties | 3 +++ core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 6 ++++-- .../src/test/java/org/apache/kylin/cube/CubeManagerTest.java | 8 ++++++++ examples/test_case_data/localmeta/kylin.properties | 6 ++++++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index ff76be2..5a10660 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -716,6 +716,10 @@ abstract public class KylinConfigBase implements Serializable { return StorageURL.valueOf(url); } + public String getCubePrefixOfHTable() { + return getOptional("kylin.storage.hbase.tableprefix", "KYLIN"); + } + public String getHBaseStorageNameSpace() { return getOptional("kylin.storage.hbase.namespace", "default"); } diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties index cf0d226..66d3772 100644 --- a/core-common/src/main/resources/kylin-defaults.properties +++ b/core-common/src/main/resources/kylin-defaults.properties @@ -71,6 +71,9 @@ kylin.source.hive.redistribute-flat-table=true # The storage for final cube file in hbase kylin.storage.url=hbase +# The prefix for cube table +kylin.storage.hbase.tableprefix=KYLIN_ + # The namespace for hbase storage kylin.storage.hbase.namespace=default diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 606ce4d..a0a0eba 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -35,6 +35,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.KylinConfigExt; @@ -683,9 +684,10 @@ public class CubeManager implements IRealizationProvider { return segment; } - private String generateStorageLocation() { + @VisibleForTesting + /*private*/ String generateStorageLocation() { String namespace = config.getHBaseStorageNameSpace(); - String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix; + String namePrefix = config.getCubePrefixOfHTable(); String tableName = ""; Random ran = new Random(); do { diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index 7dd7212..da27032 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -305,6 +305,14 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { assertTrue((Long) mergedSeg.start.v == 0 && (Long) mergedSeg.end.v == 8000); } + @Test + public void testGetCubeNameWithNamespace() { + CubeManager mgr = CubeManager.getInstance(getTestConfig()); + String tablename = mgr.generateStorageLocation(); + System.out.println(tablename); + assertTrue(tablename.startsWith("HELLO:WORLD")); + } + public CubeDescManager getCubeDescManager() { return CubeDescManager.getInstance(getTestConfig()); } diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index c7dda3f..48e2856 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -39,6 +39,12 @@ kylin.metadata.url= # The storage for final cube file in hbase kylin.storage.url=hbase +# The prefix for cube table +kylin.storage.hbase.tableprefix=WORLD_ + +# The namespace for hbase storage +kylin.storage.hbase.namespace=HELLO + # Working folder in HDFS, make sure user has the right access to the hdfs directory kylin.env.hdfs-working-dir=/kylin -- 2.4.9 (Apple Git-60) From c0afa3e518ee3759273ea32999c1e8a859f878e7 Mon Sep 17 00:00:00 2001 From: Albert Lee Date: Sun, 10 Sep 2017 23:29:25 +0800 Subject: [PATCH 2/3] Add function for cube hbase tablename prefix. --- core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 5a10660..f56e2f3 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -717,7 +717,7 @@ abstract public class KylinConfigBase implements Serializable { } public String getCubePrefixOfHTable() { - return getOptional("kylin.storage.hbase.tableprefix", "KYLIN"); + return getOptional("kylin.storage.hbase.tableprefix", "KYLIN_"); } public String getHBaseStorageNameSpace() { -- 2.4.9 (Apple Git-60) From 3bbaea3704fdf646f4db27996e852adef57d6cba Mon Sep 17 00:00:00 2001 From: Albert Lee Date: Mon, 11 Sep 2017 02:34:21 +0800 Subject: [PATCH 3/3] Fix a code style problem. --- core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index a0a0eba..7ee6c93 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -66,7 +66,6 @@ import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.IRealizationConstants; import org.apache.kylin.metadata.realization.IRealizationProvider; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.metadata.realization.RealizationType; @@ -692,14 +691,12 @@ public class CubeManager implements IRealizationProvider { Random ran = new Random(); do { StringBuffer sb = new StringBuffer(); - sb.append(namespace).append(":"); - sb.append(namePrefix); + sb.append(namespace).append(":").append(namePrefix); for (int i = 0; i < HBASE_TABLE_LENGTH; i++) { sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length()))); } tableName = sb.toString(); } while (this.usedStorageLocation.containsValue(tableName)); - return tableName; } -- 2.4.9 (Apple Git-60)