From c798528b6b34341628b582e3a5b9d2f3b75c4e93 Mon Sep 17 00:00:00 2001 From: "peng.jianhua" Date: Thu, 25 Jan 2018 09:23:39 +0800 Subject: [PATCH 1/1] KYLIN-3086 When you load Hive table metadata from server, the intermediate tables should not be loaded. --- .../org/apache/kylin/common/KylinConstant.java | 24 ++++++++++++++++++++++ .../kylin/cube/model/CubeJoinedFlatTableDesc.java | 5 +++-- .../org/apache/kylin/metadata/model/TableDesc.java | 3 ++- .../inmemcubing/ITDoggedCubeBuilderStressTest.java | 4 +++- .../cube/inmemcubing/ITDoggedCubeBuilderTest.java | 4 +++- .../cube/inmemcubing/ITInMemCubeBuilderTest.java | 9 ++++++-- .../apache/kylin/rest/job/StorageCleanupJob.java | 6 ++++-- .../apache/kylin/rest/service/TableService.java | 14 ++++++++++++- .../storage/hbase/util/StorageCleanupJob.java | 5 +++-- 9 files changed, 62 insertions(+), 12 deletions(-) create mode 100644 core-common/src/main/java/org/apache/kylin/common/KylinConstant.java diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java b/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java new file mode 100644 index 0000000..d8073b7 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.common; + +public class KylinConstant { + + public static final String KYLIN_INTERMEDIATE_PREFIX = "kylin_intermediate_"; + +} \ No newline at end of file diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java index 81087f8..231cd54 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java @@ -22,6 +22,7 @@ import java.io.Serializable; import java.util.List; import java.util.Map; +import org.apache.kylin.common.KylinConstant; import org.apache.kylin.common.util.BytesSplitter; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.metadata.model.DataModelDesc; @@ -74,9 +75,9 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, Serializab protected String makeTableName(CubeDesc cubeDesc, CubeSegment cubeSegment) { if (cubeSegment == null) { - return "kylin_intermediate_" + cubeDesc.getName().toLowerCase(); + return KylinConstant.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase(); } else { - return "kylin_intermediate_" + cubeDesc.getName().toLowerCase() + "_" + return KylinConstant.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase() + "_" + cubeSegment.getUuid().replaceAll("-", "_"); } } diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java index e456d9a..f9d549b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; +import org.apache.kylin.common.KylinConstant; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.Pair; @@ -45,7 +46,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { private static final Logger logger = LoggerFactory.getLogger(TableDesc.class); private static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW"; - private static final String materializedTableNamePrefix = "kylin_intermediate_"; + private static final String materializedTableNamePrefix = KylinConstant.KYLIN_INTERMEDIATE_PREFIX; public static String concatRawResourcePath(String nameOnPath) { return ResourceStore.TABLE_RESOURCE_ROOT + "/" + nameOnPath + ".json"; diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java index fa2d792..f56beb0 100644 --- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java @@ -26,6 +26,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.KylinConstant; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeInstance; @@ -66,7 +67,8 @@ public class ITDoggedCubeBuilderStressTest extends LocalFileMetadataTestCase { CubeManager cubeManager = CubeManager.getInstance(kylinConfig); cube = cubeManager.getCube("ssb"); - flatTable = LOCALMETA_TEST_DATA + "/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv"; + flatTable = LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX + + "ssb_19920101000000_19920201000000.csv"; dictionaryMap = ITInMemCubeBuilderTest.getDictionaryMap(cube, flatTable); } diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java index 865cdbb..52a1c6d 100644 --- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java @@ -33,6 +33,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.KylinConstant; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeInstance; @@ -70,7 +71,8 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase { CubeManager cubeManager = CubeManager.getInstance(kylinConfig); cube = cubeManager.getCube("ssb"); - flatTable = LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv"; + flatTable = LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX + + "ssb_19920101000000_19920201000000.csv"; dictionaryMap = ITInMemCubeBuilderTest.getDictionaryMap(cube, flatTable); } diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java index f1b65b0..77a35a2 100644 --- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java @@ -33,6 +33,7 @@ import java.util.concurrent.Future; import org.apache.commons.io.FileUtils; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.KylinConstant; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeInstance; @@ -83,13 +84,17 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase { @Test public void testSSBCubeMore() throws Exception { testBuild("ssb", // - LOCALMETA_TEST_DATA + "/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv", 7000, 4); + LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX + + "ssb_19920101000000_19920201000000.csv", + 7000, 4); } @Test public void testSSBCube() throws Exception { testBuild("ssb", // - LOCALMETA_TEST_DATA + "/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv", 1000, 1); + LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX + + "ssb_19920101000000_19920201000000.csv", + 1000, 1); } public void testBuild(String cubeName, String flatTable, int nInpRows, int nThreads) throws Exception { diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java index d1cb176..189e827 100755 --- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java +++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java @@ -38,6 +38,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.KylinConstant; +import org.apache.kylin.common.metrics.common.MetricsConstant; import org.apache.kylin.common.util.AbstractApplication; import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.common.util.HadoopUtil; @@ -192,7 +194,7 @@ public class StorageCleanupJob extends AbstractApplication { JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv()); final CliCommandExecutor cmdExec = config.getCliCommandExecutor(); final int uuidLength = 36; - final String preFix = "kylin_intermediate_"; + final String preFix = KylinConstant.KYLIN_INTERMEDIATE_PREFIX; final String uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; ISourceMetadataExplorer explr = SourceFactory.getDefaultSource().getSourceMetadataExplorer(); @@ -200,7 +202,7 @@ public class StorageCleanupJob extends AbstractApplication { Iterable kylinIntermediates = Iterables.filter(hiveTableNames, new Predicate() { @Override public boolean apply(@Nullable String input) { - return input != null && input.startsWith("kylin_intermediate_"); + return input != null && input.startsWith(KylinConstant.KYLIN_INTERMEDIATE_PREFIX); } }); diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java index 8a6dd96..caa7130 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -29,7 +29,10 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import javax.annotation.Nullable; + import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConstant; import org.apache.kylin.common.util.HadoopUtil; import org.apache.kylin.common.util.Pair; import org.apache.kylin.cube.CubeManager; @@ -61,6 +64,8 @@ import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.SetMultimap; @@ -324,7 +329,14 @@ public class TableService extends BasicService { */ public List getHiveTableNames(String database) throws Exception { ISourceMetadataExplorer explr = SourceFactory.getDefaultSource().getSourceMetadataExplorer(); - return explr.listTables(database); + List hiveTableNames = explr.listTables(database); + Iterable kylinApplicationTableNames = Iterables.filter(hiveTableNames, new Predicate() { + @Override + public boolean apply(@Nullable String input) { + return input != null && !input.startsWith(KylinConstant.KYLIN_INTERMEDIATE_PREFIX); + } + }); + return Lists.newArrayList(kylinApplicationTableNames); } private TableDescResponse cloneTableDesc(TableDesc table, String prj) { diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java index 2ae21b4..f81de6c 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.KylinConstant; import org.apache.kylin.common.util.AbstractApplication; import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.common.util.HadoopUtil; @@ -251,13 +252,13 @@ public class StorageCleanupJob extends AbstractApplication { final KylinConfig config = KylinConfig.getInstanceFromEnv(); final CliCommandExecutor cmdExec = config.getCliCommandExecutor(); final int uuidLength = 36; - final String preFix = "kylin_intermediate_"; + final String preFix = KylinConstant.KYLIN_INTERMEDIATE_PREFIX; final String uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; final String useDatabaseHql = "USE " + config.getHiveDatabaseForIntermediateTable() + ";"; final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder(); hiveCmdBuilder.addStatement(useDatabaseHql); - hiveCmdBuilder.addStatement("show tables " + "\'kylin_intermediate_*\'" + "; "); + hiveCmdBuilder.addStatement("show tables " + "\'" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX + "*\'" + "; "); Pair result = cmdExec.execute(hiveCmdBuilder.build()); -- 2.7.2.windows.1