From aec6481a482c4b554b8246906a61d014468a2941 Mon Sep 17 00:00:00 2001 From: Zhong Date: Wed, 13 Jul 2016 18:04:48 +0800 Subject: [PATCH 1/3] Change comment for hierarchy dimensions --- .../src/main/java/org/apache/kylin/cube/model/AggregationGroup.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index 0e25ac7..601ee0a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -36,7 +36,7 @@ public class AggregationGroup { public static class HierarchyMask { public long fullMask;// 00000111 public long[] allMasks;// 00000100,00000110,00000111 - public long[] dims;// 00000100,00000100,00000001 + public long[] dims;// 00000100,00000010,00000001 } @JsonProperty("includes") -- 2.5.4 (Apple Git-61) From a072a6bda1a07dba1e078d173b3e366605a67f05 Mon Sep 17 00:00:00 2001 From: Zhong Date: Wed, 13 Jul 2016 18:26:07 +0800 Subject: [PATCH 2/3] KYLIN-1883: Consensus Problem when running the tool, MetadataCleanupJob --- .../kylin/engine/mr/steps/MetadataCleanupJob.java | 32 ++++++++++------------ 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java index f300de9..f8d265b 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java @@ -101,15 +101,6 @@ public class MetadataCleanupJob extends AbstractHadoopJob { public void cleanup() throws Exception { CubeManager cubeManager = CubeManager.getInstance(config); - Set activeResourceList = Sets.newHashSet(); - for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) { - for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) { - activeResourceList.addAll(segment.getSnapshotPaths()); - activeResourceList.addAll(segment.getDictionaryPaths()); - activeResourceList.add(segment.getStatisticsResourcePath()); - } - } - List toDeleteResource = Lists.newArrayList(); // two level resources, snapshot tables and cube statistics @@ -121,10 +112,8 @@ public class MetadataCleanupJob extends AbstractHadoopJob { NavigableSet snapshotNames = getStore().listResources(snapshotTable); if (snapshotNames != null) for (String snapshot : snapshotNames) { - if (!activeResourceList.contains(snapshot)) { - if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot))) - toDeleteResource.add(snapshot); - } + if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot))) + toDeleteResource.add(snapshot); } } } @@ -140,13 +129,22 @@ public class MetadataCleanupJob extends AbstractHadoopJob { NavigableSet dictionaries = getStore().listResources(tableCol); if (dictionaries != null) for (String dict : dictionaries) - if (!activeResourceList.contains(dict)) { - if (isOlderThanThreshold(getStore().getResourceTimestamp(dict))) - toDeleteResource.add(dict); - } + if (isOlderThanThreshold(getStore().getResourceTimestamp(dict))) + toDeleteResource.add(dict); } } + Set activeResourceList = Sets.newHashSet(); + for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) { + for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) { + activeResourceList.addAll(segment.getSnapshotPaths()); + activeResourceList.addAll(segment.getDictionaryPaths()); + activeResourceList.add(segment.getStatisticsResourcePath()); + } + } + + toDeleteResource.removeAll(activeResourceList); + // delete old and completed jobs ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv()); List allExecutable = executableDao.getJobs(); -- 2.5.4 (Apple Git-61) From b2d296588b0e7824427cbece711f7236c996bb45 Mon Sep 17 00:00:00 2001 From: Zhong Date: Thu, 14 Jul 2016 14:59:12 +0800 Subject: [PATCH 3/3] Remove nonexistent task items for EXECUTE_RESOURCE_ROOT from toDeleteResource --- .../main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java | 1 - 1 file changed, 1 deletion(-) diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java index f8d265b..dd482d6 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java @@ -156,7 +156,6 @@ public class MetadataCleanupJob extends AbstractHadoopJob { toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid()); for (ExecutablePO task : executable.getTasks()) { - toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid()); toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid()); } } -- 2.5.4 (Apple Git-61)