From 994bbcfdf6d1dcd345693c0a5e1a03869fb251bf Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Tue, 25 Nov 2014 17:06:31 +0800 Subject: [PATCH 1/2] add uuid and create time for CubeSegment for future use and debug --- .../main/java/com/kylinolap/cube/CubeManager.java | 26 ++++++++++------------ .../main/java/com/kylinolap/cube/CubeSegment.java | 21 +++++++++++++++++ .../java/com/kylinolap/dict/DateStrDictionary.java | 2 +- .../main/java/com/kylinolap/job/JobManager.java | 8 +++---- .../com/kylinolap/job/BuildCubeWithEngineTest.java | 3 ++- .../java/com/kylinolap/job/JobInstanceTest.java | 4 +++- .../job/engine/JobInstanceBuilderTest.java | 5 +++-- .../com/kylinolap/rest/service/JobService.java | 5 +++-- 8 files changed, 49 insertions(+), 25 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java index 1c01dab..03c6933 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java @@ -17,14 +17,10 @@ package com.kylinolap.cube; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import com.kylinolap.dict.DateStrDictionary; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -535,17 +531,19 @@ private void afterCubeDroped(CubeInstance droppedCube, List pro * @return */ private CubeSegment buildSegment(CubeInstance cubeInstance, long startDate, long endDate) { - CubeSegment incrementalSeg = new CubeSegment(); + CubeSegment segment = new CubeSegment(); String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate); - incrementalSeg.setName(incrementalSegName); - incrementalSeg.setDateRangeStart(startDate); - incrementalSeg.setDateRangeEnd(endDate); - incrementalSeg.setStatus(CubeSegmentStatusEnum.NEW); - incrementalSeg.setStorageLocationIdentifier(generateStorageLocation()); + segment.setUuid(UUID.randomUUID().toString()); + segment.setName(incrementalSegName); + segment.setCreateTime(DateStrDictionary.dateToString(new Date())); + segment.setDateRangeStart(startDate); + segment.setDateRangeEnd(endDate); + segment.setStatus(CubeSegmentStatusEnum.NEW); + segment.setStorageLocationIdentifier(generateStorageLocation()); - incrementalSeg.setCubeInstance(cubeInstance); + segment.setCubeInstance(cubeInstance); - return incrementalSeg; + return segment; } private String generateStorageLocation() { diff --git a/cube/src/main/java/com/kylinolap/cube/CubeSegment.java b/cube/src/main/java/com/kylinolap/cube/CubeSegment.java index 6e34aa5..aa841d8 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeSegment.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeSegment.java @@ -33,6 +33,8 @@ @JsonBackReference private CubeInstance cubeInstance; + @JsonProperty("uuid") + private String uuid; @JsonProperty("name") private String name; @JsonProperty("storage_location_identifier") @@ -53,6 +55,8 @@ private long lastBuildTime; @JsonProperty("last_build_job_id") private String lastBuildJobID; + @JsonProperty("create_time") + private String createTime; @JsonProperty("binary_signature") private String binarySignature; // a hash of cube schema and dictionary ID, @@ -90,6 +94,15 @@ public static String getSegmentName(long startDate, long endDate) { // ============================================================================ + + public String getUuid() { + return uuid; + } + + public void setUuid(String id) { + this.uuid = id; + } + public String getName() { return name; } @@ -162,6 +175,14 @@ public void setLastBuildJobID(String lastBuildJobID) { this.lastBuildJobID = lastBuildJobID; } + public String getCreateTime() { + return createTime; + } + + public void setCreateTime(String createTime) { + this.createTime = createTime; + } + public String getBinarySignature() { return binarySignature; } diff --git a/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java b/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java index 1114546..39af590 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java +++ b/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java @@ -66,7 +66,7 @@ static SimpleDateFormat getDateFormat(String datePattern) { } public static String dateToString(Date date) { - return dateToString(date, DEFAULT_DATE_PATTERN); + return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS); } public static String dateToString(Date date, String pattern) { diff --git a/job/src/main/java/com/kylinolap/job/JobManager.java b/job/src/main/java/com/kylinolap/job/JobManager.java index 2f1107f..a95d94c 100644 --- a/job/src/main/java/com/kylinolap/job/JobManager.java +++ b/job/src/main/java/com/kylinolap/job/JobManager.java @@ -72,9 +72,9 @@ public JobManager(String engineID, JobEngineConfig engineCfg) throws JobExceptio this.jobEngine = JobEngine.getInstance(engineID, engineCfg); } - public JobInstance createJob(String cubeName, String segmentName, CubeBuildTypeEnum jobType) throws IOException { + public JobInstance createJob(String cubeName, String segmentName, String uuid, CubeBuildTypeEnum jobType) throws IOException { // build job instance - JobInstance jobInstance = buildJobInstance(cubeName, segmentName, jobType); + JobInstance jobInstance = buildJobInstance(cubeName, segmentName, uuid, jobType); // create job steps based on job type JobInstanceBuilder stepBuilder = new JobInstanceBuilder(this.engineConfig); @@ -83,12 +83,12 @@ public JobInstance createJob(String cubeName, String segmentName, CubeBuildTypeE return jobInstance; } - private JobInstance buildJobInstance(String cubeName, String segmentName, CubeBuildTypeEnum jobType) { + private JobInstance buildJobInstance(String cubeName, String segmentName, String uuid, CubeBuildTypeEnum jobType) { SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone(this.engineConfig.getTimeZone())); JobInstance jobInstance = new JobInstance(); - jobInstance.setUuid(UUID.randomUUID().toString()); + jobInstance.setUuid(uuid); jobInstance.setType(jobType); jobInstance.setRelatedCube(cubeName); jobInstance.setRelatedSegment(segmentName); diff --git a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java index f0b7b1f..3db4923 100644 --- a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java +++ b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.List; import java.util.TimeZone; +import java.util.UUID; import org.junit.After; import org.junit.Before; @@ -200,7 +201,7 @@ protected void waitCubeBuilt(List jobs) throws Exception { System.out.println(JsonUtil.writeValueAsIndentString(cube)); for (CubeSegment seg : newSegments) { - JobInstance newJob = jobManager.createJob(cubename, seg.getName(), jobType); + JobInstance newJob = jobManager.createJob(cubename, seg.getName(), UUID.randomUUID().toString(), jobType); // submit job to store String jobUuid = jobManager.submitJob(newJob); jobList.add(jobUuid); diff --git a/job/src/test/java/com/kylinolap/job/JobInstanceTest.java b/job/src/test/java/com/kylinolap/job/JobInstanceTest.java index dcffa10..686f65e 100644 --- a/job/src/test/java/com/kylinolap/job/JobInstanceTest.java +++ b/job/src/test/java/com/kylinolap/job/JobInstanceTest.java @@ -30,6 +30,8 @@ import com.kylinolap.job.constant.JobStepStatusEnum; import com.kylinolap.job.engine.JobEngineConfig; +import java.util.UUID; + /** * @author ysong1 * @@ -50,7 +52,7 @@ public void testJobInstanceStatus() throws Exception { KylinConfig kylinCfg = KylinConfig.getInstanceFromEnv(); JobManager jobManager = new JobManager("JobInstanceTest", new JobEngineConfig(kylinCfg)); - JobInstance jobInstance = jobManager.createJob("test_kylin_cube_with_slr_1_new_segment", "20130331080000_20131212080000", CubeBuildTypeEnum.BUILD); + JobInstance jobInstance = jobManager.createJob("test_kylin_cube_with_slr_1_new_segment", "20130331080000_20131212080000", UUID.randomUUID().toString(), CubeBuildTypeEnum.BUILD); // initial job status should be PENDING assertEquals(JobStatusEnum.PENDING, jobInstance.getStatus()); diff --git a/job/src/test/java/com/kylinolap/job/engine/JobInstanceBuilderTest.java b/job/src/test/java/com/kylinolap/job/engine/JobInstanceBuilderTest.java index a412bf5..0734587 100644 --- a/job/src/test/java/com/kylinolap/job/engine/JobInstanceBuilderTest.java +++ b/job/src/test/java/com/kylinolap/job/engine/JobInstanceBuilderTest.java @@ -19,6 +19,7 @@ import java.text.SimpleDateFormat; import java.util.TimeZone; +import java.util.UUID; import org.junit.After; import org.junit.Before; @@ -75,7 +76,7 @@ public void testCreateSteps() throws Exception { // initial segment CubeSegment segment = cubeManager.allocateSegments(cube, CubeBuildTypeEnum.BUILD, 0, dateEnd).get(0); - JobInstance jobInstance = jobManager.createJob(cubeName, segment.getName(), CubeBuildTypeEnum.BUILD); + JobInstance jobInstance = jobManager.createJob(cubeName, segment.getName(), UUID.randomUUID().toString(), CubeBuildTypeEnum.BUILD); String actual = JsonUtil.writeValueAsIndentString(jobInstance); System.out.println(actual); @@ -150,7 +151,7 @@ public void testCreateMergeSteps() throws Exception { // initial segment CubeSegment segment = CubeManager.getInstance(this.getTestConfig()).allocateSegments(cube, CubeBuildTypeEnum.MERGE, 1384240200000L, 1386835200000L).get(0); - JobInstance jobInstance = jobManager.createJob(cubeName, segment.getName(), CubeBuildTypeEnum.MERGE); + JobInstance jobInstance = jobManager.createJob(cubeName, segment.getName(), UUID.randomUUID().toString(), CubeBuildTypeEnum.MERGE); String actual = JsonUtil.writeValueAsIndentString(jobInstance); System.out.println(actual); diff --git a/server/src/main/java/com/kylinolap/rest/service/JobService.java b/server/src/main/java/com/kylinolap/rest/service/JobService.java index 815f480..c9e2f45 100644 --- a/server/src/main/java/com/kylinolap/rest/service/JobService.java +++ b/server/src/main/java/com/kylinolap/rest/service/JobService.java @@ -107,8 +107,9 @@ public String submitJob(CubeInstance cube, long startDate, long endDate, CubeBui String uuid = null; try { for (CubeSegment segment : this.getCubeManager().allocateSegments(cube, buildType, startDate, endDate)) { - JobInstance job = this.getJobManager().createJob(cube.getName(), segment.getName(), buildType); - uuid = this.getJobManager().submitJob(job); + uuid = segment.getUuid(); + JobInstance job = this.getJobManager().createJob(cube.getName(), segment.getName(), uuid, buildType); + this.getJobManager().submitJob(job); permissionService.init(job, null); permissionService.inherit(job, cube); } From f2bcf062932977a15647ea6864f657871e6bacbf Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Wed, 26 Nov 2014 17:06:56 +0800 Subject: [PATCH 2/2] fix CI issue --- .../com/kylinolap/job/BuildCubeWithEngineTest.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java index a9ad637..5f031f2 100644 --- a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java +++ b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java @@ -26,6 +26,7 @@ import java.util.TimeZone; import java.util.UUID; +import com.google.common.collect.Lists; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -225,7 +226,6 @@ protected void waitCubeBuilt(List jobs) throws Exception { } protected List submitJob(String cubename, long startDate, long endDate, CubeBuildTypeEnum jobType) throws SchedulerException, IOException, InvalidJobInstanceException, CubeIntegrityException { - List jobList = new ArrayList(); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeInstance cube = cubeMgr.getCube(cubename); @@ -235,13 +235,20 @@ protected void waitCubeBuilt(List jobs) throws Exception { List newSegments = cubeMgr.allocateSegments(cube, jobType, startDate, endDate); System.out.println(JsonUtil.writeValueAsIndentString(cube)); + List jobUuids = Lists.newArrayList(); + List jobs = Lists.newArrayList(); for (CubeSegment seg : newSegments) { - JobInstance newJob = jobManager.createJob(cubename, seg.getName(), UUID.randomUUID().toString(), jobType); + String uuid = seg.getUuid(); + seg.setLastBuildJobID(uuid); + + jobUuids.add(uuid); + jobs.add(jobManager.createJob(cubename, seg.getName(), uuid, jobType)); // submit job to store - String jobUuid = jobManager.submitJob(newJob); - jobList.add(jobUuid); } - - return jobList; + cubeMgr.updateCube(cube); + for (JobInstance job: jobs) { + jobManager.submitJob(job); + } + return jobUuids; } } \ No newline at end of file