From f5e8741074aa7c4446221516b353b53e3f64f9e7 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Fri, 9 Jan 2015 10:10:51 +0800 Subject: [PATCH 1/3] 1. fix cube date\n2. fix spring context --- .../java/com/kylinolap/job/BuildCubeWithEngineTest.java | 2 +- server/src/main/resources/applicationContext.xml | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java index ce19a62..c2a61d9 100644 --- a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java +++ b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java @@ -108,7 +108,7 @@ public void testBuild() throws Exception { waitForJob(job.getId()); assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState()); - Date date2 = dateFormat.parse("2013-04-01"); + Date date2 = dateFormat.parse("2015-01-01"); final CubeSegment cubeSegment2 = cubeManager.appendSegments(testCube, date1.getTime(), date2.getTime()); final CubingJob job2 = CubingJobBuilder.newBuilder().setJobEnginConfig(jobEngineConfig).setSegment(cubeSegment2).buildJob(); jobService.addJob(job2); diff --git a/server/src/main/resources/applicationContext.xml b/server/src/main/resources/applicationContext.xml index 2694b75..5de7fe5 100644 --- a/server/src/main/resources/applicationContext.xml +++ b/server/src/main/resources/applicationContext.xml @@ -30,11 +30,7 @@ - - - - - + @@ -43,10 +39,10 @@ - - + + + + From 2e3effc497d81a192b33fc326812df0e25651449 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Fri, 9 Jan 2015 17:24:26 +0800 Subject: [PATCH 2/3] refactor BuildCubeWithEngine --- .../com/kylinolap/cube/CubeSegmentValidator.java | 2 +- .../java/com/kylinolap/job/JoinedFlatTable.java | 2 +- .../kylinolap/job2/service/ExecutableManager.java | 15 +- .../com/kylinolap/job/BuildCubeWithEngineTest.java | 172 +++++++++++++++++---- 4 files changed, 152 insertions(+), 39 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java index e263be9..ef93b05 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java @@ -152,7 +152,7 @@ void validate(CubeInstance cubeInstance, CubeSegment newSegment) throws CubeInte throw new CubeIntegrityException("there is gap in cube segments"); } } - if (newSegment.getDateRangeStart() == initStartDate && startDate < newSegment.getDateRangeEnd()) { + if (newSegment.getDateRangeStart() == startDate && startDate < newSegment.getDateRangeEnd()) { return; } throw new CubeIntegrityException("invalid segment date range from " + newSegment.getDateRangeStart() + " to " + newSegment.getDateRangeEnd()); diff --git a/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java b/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java index 8767e12..e05da9e 100644 --- a/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java +++ b/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java @@ -198,7 +198,7 @@ private static void appendWhereStatement(IJoinedFlatTableDesc intermediateTableD long dateStart = cubeSegment.getDateRangeStart(); long dateEnd = cubeSegment.getDateRangeEnd(); - if (!(dateStart == 0 && dateEnd == 0)) { + if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) { String partitionColumnName = cubeDesc.getCubePartitionDesc().getPartitionDateColumn(); int indexOfDot = partitionColumnName.lastIndexOf("."); diff --git a/job/src/main/java/com/kylinolap/job2/service/ExecutableManager.java b/job/src/main/java/com/kylinolap/job2/service/ExecutableManager.java index 8d88224..a27f829 100644 --- a/job/src/main/java/com/kylinolap/job2/service/ExecutableManager.java +++ b/job/src/main/java/com/kylinolap/job2/service/ExecutableManager.java @@ -3,6 +3,7 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.kylinolap.common.KylinConfig; import com.kylinolap.job2.dao.JobDao; import com.kylinolap.job2.dao.JobOutputPO; @@ -94,6 +95,7 @@ public AbstractExecutable getJob(String uuid) { public Output getOutput(String uuid) { try { final JobOutputPO jobOutput = jobDao.getJobOutput(uuid); + Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + uuid); final DefaultOutput result = new DefaultOutput(); result.setExtra(jobOutput.getInfo()); result.setState(ExecutableState.valueOf(jobOutput.getStatus())); @@ -175,6 +177,7 @@ public void discardJob(String jobId) { public void updateJobOutput(String jobId, ExecutableState newStatus, Map info, String output) { try { final JobOutputPO jobOutput = jobDao.getJobOutput(jobId); + Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + jobId); ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus()); if (newStatus != null && oldStatus != newStatus) { if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) { @@ -243,6 +246,7 @@ public void addJobInfo(String id, Map info) { } try { JobOutputPO output = jobDao.getJobOutput(id); + Preconditions.checkArgument(output != null, "there is no related output for job id:" + id); output.getInfo().putAll(info); jobDao.updateJobOutput(output); } catch (PersistentException e) { @@ -252,14 +256,9 @@ public void addJobInfo(String id, Map info) { } public void addJobInfo(String id, String key, String value) { - try { - JobOutputPO output = jobDao.getJobOutput(id); - output.getInfo().put(key, value); - jobDao.updateJobOutput(output); - } catch (PersistentException e) { - logger.error("error update job info, id:" + id + " key:" + key + " value:" + value); - throw new RuntimeException(e); - } + Map info = Maps.newHashMap(); + info.put(key, value); + addJobInfo(id, info); } private void stopJob(AbstractExecutable job) { diff --git a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java index c2a61d9..811b313 100644 --- a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java +++ b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java @@ -1,5 +1,6 @@ package com.kylinolap.job; +import com.google.common.collect.Lists; import com.kylinolap.common.KylinConfig; import com.kylinolap.common.util.AbstractKylinTestCase; import com.kylinolap.common.util.ClasspathUtil; @@ -7,6 +8,7 @@ import com.kylinolap.cube.CubeInstance; import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.CubeSegment; +import com.kylinolap.cube.model.CubeBuildTypeEnum; import com.kylinolap.job.constant.JobConstants; import com.kylinolap.job.engine.JobEngineConfig; import com.kylinolap.job.hadoop.cube.StorageCleanupJob; @@ -22,9 +24,13 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.List; +import java.util.TimeZone; +import java.util.concurrent.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -85,42 +91,150 @@ public void before() throws Exception { @After public void after() throws Exception { - int exitCode = cleanupOldCubes(); - if (exitCode == 0) { - exportHBaseData(); - } - - HBaseMetadataTestCase.staticCleanupTestMetadata(); +// int exitCode = cleanupOldCubes(); +// if (exitCode == 0) { +// exportHBaseData(); +// } +// +// HBaseMetadataTestCase.staticCleanupTestMetadata(); } @Test public void testBuild() throws Exception { - final String testCubeName = "test_kylin_cube_without_slr_left_join_empty"; - final CubeInstance testCube = cubeManager.getCube(testCubeName); - testCube.getSegments().clear(); - cubeManager.updateCube(testCube); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - Date date1 = dateFormat.parse("2013-01-01"); - final CubeSegment cubeSegment1 = cubeManager.appendSegments(testCube, 0, date1.getTime()); - final CubingJobBuilder cubingJobBuilder = CubingJobBuilder.newBuilder().setJobEnginConfig(jobEngineConfig).setSegment(cubeSegment1); - final CubingJob job = cubingJobBuilder.buildJob(); - jobService.addJob(job); - waitForJob(job.getId()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState()); + DeployUtil.prepareTestData("inner", "test_kylin_cube_with_slr_empty"); + DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty"); + + String[] testCase = new String[]{ + "testInnerJoinCube", + "testInnerJoinCube2", + "testLeftJoinCube", + "testLeftJoinCube2", + }; + ExecutorService executorService = Executors.newFixedThreadPool(testCase.length); + final CountDownLatch countDownLatch = new CountDownLatch(testCase.length); + List>> tasks = Lists.newArrayListWithExpectedSize(testCase.length); + for (int i = 0; i < testCase.length; i++) { + tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch))); + } + countDownLatch.await(); + for (int i = 0; i < tasks.size(); ++i) { + Future> task = tasks.get(i); + final List jobIds = task.get(); + for (String jobId: jobIds) { + assertJobSucceed(jobId); + } + } + } + + private void assertJobSucceed(String jobId) { + assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState()); + } + + private class TestCallable implements Callable> { + + private final String methodName; + private final CountDownLatch countDownLatch; + + public TestCallable(String methodName, CountDownLatch countDownLatch) { + this.methodName = methodName; + this.countDownLatch = countDownLatch; + } + + @Override + public List call() throws Exception { + try { + final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName); + method.setAccessible(true); + return (List) method.invoke(BuildCubeWithEngineTest.this); + } finally { + countDownLatch.countDown(); + } + } + } + + private List testInnerJoinCube2() throws Exception { + clearSegment("test_kylin_cube_with_slr_empty"); + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); + f.setTimeZone(TimeZone.getTimeZone("GMT")); + long date1 = 0; + long date2 = f.parse("2013-01-01").getTime(); + long date3 = f.parse("2022-01-01").getTime(); + List result = Lists.newArrayList(); + result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2)); + result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3)); + return result; + } - Date date2 = dateFormat.parse("2015-01-01"); - final CubeSegment cubeSegment2 = cubeManager.appendSegments(testCube, date1.getTime(), date2.getTime()); - final CubingJob job2 = CubingJobBuilder.newBuilder().setJobEnginConfig(jobEngineConfig).setSegment(cubeSegment2).buildJob(); - jobService.addJob(job2); - waitForJob(job2.getId()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job2.getId()).getState()); + private List testInnerJoinCube() throws Exception { + clearSegment("test_kylin_cube_without_slr_empty"); - final CubeSegment cubeSegment3 = cubeManager.mergeSegments(testCube, 0, date2.getTime()); - final CubingJob job3 = CubingJobBuilder.newBuilder().setJobEnginConfig(jobEngineConfig).setSegment(cubeSegment3).mergeJob(); - jobService.addJob(job3); - waitForJob(job3.getId()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job3.getId()).getState()); + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); + f.setTimeZone(TimeZone.getTimeZone("GMT")); + + // this cube's start date is 0, end date is 20501112000000 + long date1 = 0; + long date2 = f.parse("2013-01-01").getTime(); + + + // this cube doesn't support incremental build, always do full build + + List result = Lists.newArrayList(); + result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2)); + return result; + } + + private List testLeftJoinCube2() throws Exception { + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); + f.setTimeZone(TimeZone.getTimeZone("GMT")); + List result = Lists.newArrayList(); + final String cubeName = "test_kylin_cube_without_slr_left_join_empty"; + // this cube's start date is 0, end date is 20120601000000 + long dateStart = cubeManager.getCube(cubeName).getDescriptor().getCubePartitionDesc().getPartitionDateStart(); + long dateEnd = f.parse("2012-06-01").getTime(); + + clearSegment(cubeName); + result.add(buildSegment(cubeName, dateStart, dateEnd)); + + // then submit an append job, start date is 20120601000000, end + // date is 20220101000000 + dateStart = f.parse("2012-06-01").getTime(); + dateEnd = f.parse("2022-01-01").getTime(); + result.add(buildSegment(cubeName, dateStart, dateEnd)); + return result; + + } + + private List testLeftJoinCube() throws Exception { + String cubeName = "test_kylin_cube_with_slr_left_join_empty"; + clearSegment(cubeName); + + SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd"); + f.setTimeZone(TimeZone.getTimeZone("GMT")); + long dateStart = cubeManager.getCube(cubeName).getDescriptor().getCubePartitionDesc().getPartitionDateStart(); + long dateEnd = f.parse("2050-11-12").getTime(); + + // this cube's start date is 0, end date is 20501112000000 + List result = Lists.newArrayList(); + result.add(buildSegment(cubeName, dateStart, dateEnd)); + return result; + + } + + private void clearSegment(String cubeName) throws Exception{ + CubeInstance cube = cubeManager.getCube(cubeName); + cube.getSegments().clear(); + cubeManager.updateCube(cube); + } + + + private String buildSegment(String cubeName, long startDate, long endDate) throws Exception { + CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), startDate, endDate); + CubingJobBuilder cubingJobBuilder = CubingJobBuilder.newBuilder().setJobEnginConfig(jobEngineConfig).setSegment(segment); + CubingJob job = cubingJobBuilder.buildJob(); + jobService.addJob(job); + waitForJob(job.getId()); + return job.getId(); } private int cleanupOldCubes() throws Exception { From 3d055eb5ed99aa83c251fc837d041cc4f4df3517 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Fri, 9 Jan 2015 17:27:18 +0800 Subject: [PATCH 3/3] refactor --- .../test/java/com/kylinolap/job/BuildCubeWithEngineTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java index 811b313..1460818 100644 --- a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java +++ b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java @@ -91,12 +91,12 @@ public void before() throws Exception { @After public void after() throws Exception { -// int exitCode = cleanupOldCubes(); -// if (exitCode == 0) { -// exportHBaseData(); -// } -// -// HBaseMetadataTestCase.staticCleanupTestMetadata(); + int exitCode = cleanupOldCubes(); + if (exitCode == 0) { + exportHBaseData(); + } + + HBaseMetadataTestCase.staticCleanupTestMetadata(); } @Test