From 29cfed1237e7a50474c9739de1b910deb08be479 Mon Sep 17 00:00:00 2001 From: fengyu Date: Thu, 7 Jul 2016 20:21:29 +0800 Subject: [PATCH 1/8] forward build job Signed-off-by: fengyu --- .../java/org/apache/kylin/cube/CubeManager.java | 24 ++++++++++++++++++++++ .../apache/kylin/cube/model/CubeBuildTypeEnum.java | 7 ++++++- .../org/apache/kylin/rest/service/JobService.java | 3 +++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 3ef800b..8d1b236 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -419,6 +419,21 @@ public class CubeManager implements IRealizationProvider { return Pair.newPair(appendSegment, mergeSegment); } + + public CubeSegment forwardSegments(CubeInstance cube, long startDate) throws IOException { + long endDate = Long.MAX_VALUE; + if(cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) { + endDate = calculateEndDateForForwardSegment(cube); + if(startDate > cube.getDescriptor().getPartitionDateStart()) { + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); + f.setTimeZone(TimeZone.getTimeZone("GMT")); + throw new IllegalArgumentException("The selected date couldn't be before than cube's start date '" + f.format(new Date(cube.getDescriptor().getPartitionDateStart())) + "'."); + } + } else { + startDate = 0L; + } + return appendSegments(cube, startDate, endDate, true, true); + } public CubeSegment appendSegments(CubeInstance cube, long endDate) throws IOException { return appendSegments(cube, endDate, true, true); @@ -541,6 +556,15 @@ public class CubeManager implements IRealizationProvider { return existing.get(existing.size() - 1).getDateRangeEnd(); } } + + private long calculateEndDateForForwardSegment(CubeInstance cube) { + List existing = cube.getSegments(); + if (existing.isEmpty()) { + return cube.getDescriptor().getPartitionDateStart(); + } else { + return existing.get(0).getDateRangeStart(); + } + } private void checkNoBuildingSegment(CubeInstance cube) { if (cube.getBuildingSegments().size() > 0) { diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java index e3ae214..fcb12c6 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeBuildTypeEnum.java @@ -35,5 +35,10 @@ public enum CubeBuildTypeEnum { /** * refresh segments */ - REFRESH + REFRESH, + /** + * Forward build segments + */ + FORWARD + } diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java index 875d7ea..8834226 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -228,6 +228,9 @@ public class JobService extends BasicService { CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, startDate, endDate); job = EngineFactory.createBatchCubingJob(refreshSeg, submitter); + } else if (buildType == CubeBuildTypeEnum.FORWARD) { + CubeSegment newSeg = getCubeManager().forwardSegments(cube, startDate); + job = EngineFactory.createBatchCubingJob(newSeg, submitter); } else { throw new JobException("invalid build type:" + buildType); } -- 1.9.4.msysgit.2