From fe45d3b04e066aae4d8f49e72d0d478dfa45fba4 Mon Sep 17 00:00:00 2001 From: honma Date: Wed, 19 Nov 2014 10:15:25 +0800 Subject: [PATCH 01/24] remove hard code /tmp/kylin/logs --- job/src/main/java/com/kylinolap/job/engine/JobEngine.java | 5 +++-- job/src/main/java/com/kylinolap/job/flow/JobFlow.java | 9 ++++++++- .../test/java/com/kylinolap/job/engine/GenericJobEngineTest.java | 5 ++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/job/src/main/java/com/kylinolap/job/engine/JobEngine.java b/job/src/main/java/com/kylinolap/job/engine/JobEngine.java index 4b525c1..b86fe36 100644 --- a/job/src/main/java/com/kylinolap/job/engine/JobEngine.java +++ b/job/src/main/java/com/kylinolap/job/engine/JobEngine.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.concurrent.ConcurrentHashMap; +import com.kylinolap.common.KylinConfig; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.math.stat.descriptive.rank.Percentile; import org.apache.curator.RetryPolicy; @@ -121,8 +122,8 @@ public void start(int daemonJobIntervalInSeconds) throws Exception { } private void startScheduler() throws JobException, IOException { - // FIXME hard code - new File("/tmp/kylin/logs/").mkdirs(); + String logDir = KylinConfig.getInstanceFromEnv().getKylinJobLogDir(); + new File(logDir).mkdirs(); log.info("Starting scheduler."); this.scheduler.start(); diff --git a/job/src/main/java/com/kylinolap/job/flow/JobFlow.java b/job/src/main/java/com/kylinolap/job/flow/JobFlow.java index 27387b8..4ba5fa3 100644 --- a/job/src/main/java/com/kylinolap/job/flow/JobFlow.java +++ b/job/src/main/java/com/kylinolap/job/flow/JobFlow.java @@ -16,10 +16,13 @@ package com.kylinolap.job.flow; +import java.io.File; +import java.io.IOException; import java.util.Collections; import java.util.LinkedList; import java.util.List; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.quartz.JobBuilder; import org.quartz.JobDetail; @@ -35,7 +38,6 @@ /** * @author xduo - * */ public class JobFlow { @@ -144,6 +146,11 @@ private String wrapExecCmd(JobInstance job, String cmd, String suffix) { return cmd; KylinConfig config = KylinConfig.getInstanceFromEnv(); + try { + FileUtils.forceMkdir(new File(config.getKylinJobLogDir())); + } catch (IOException e) { + throw new RuntimeException("Create log dir " + config.getKylinJobLogDir() + " failed.", e); + } String log = config.getKylinJobLogDir() + "/" + job.getUuid() + "_" + suffix + ".log"; return "set -o pipefail; " + cmd + " 2>&1 | tee " + log; diff --git a/job/src/test/java/com/kylinolap/job/engine/GenericJobEngineTest.java b/job/src/test/java/com/kylinolap/job/engine/GenericJobEngineTest.java index 2331cf4..ed06291 100644 --- a/job/src/test/java/com/kylinolap/job/engine/GenericJobEngineTest.java +++ b/job/src/test/java/com/kylinolap/job/engine/GenericJobEngineTest.java @@ -52,7 +52,6 @@ /** * @author ysong1 - * */ public class GenericJobEngineTest { private static String cubeName = "test_kylin_cube_with_slr_empty"; @@ -110,7 +109,7 @@ public static void scpFilesToHdfs(SSHClient hadoopCli, String[] localFiles, Stri @BeforeClass public static void beforeClass() throws Exception { - FileUtils.forceMkdir(new File("/tmp/kylin/logs/")); + FileUtils.forceMkdir(new File(KylinConfig.getInstanceFromEnv().getKylinJobLogDir())); FileUtils.deleteDirectory(new File(tempTestMetadataUrl)); FileUtils.copyDirectory(new File("../examples/test_case_data"), new File(tempTestMetadataUrl)); @@ -124,7 +123,7 @@ public static void beforeClass() throws Exception { hadoopCli.scpFileToRemote("src/test/resources/jarfile/SampleBadJavaProgram.jarfile", "/tmp"); // create log dir - hadoopCli.execCommand("mkdir -p /tmp/kylin/logs/"); + hadoopCli.execCommand("mkdir -p " + KylinConfig.getInstanceFromEnv().getKylinJobLogDir()); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); kylinConfig.setMetadataUrl(tempTestMetadataUrl); From 9b2b24da5329b82429b27814c6d89b0823405f5a Mon Sep 17 00:00:00 2001 From: honma Date: Wed, 19 Nov 2014 14:33:19 +0800 Subject: [PATCH 02/24] fix hive command no log dir bug --- job/src/main/java/com/kylinolap/job/flow/JobFlow.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/job/src/main/java/com/kylinolap/job/flow/JobFlow.java b/job/src/main/java/com/kylinolap/job/flow/JobFlow.java index 4ba5fa3..9a58107 100644 --- a/job/src/main/java/com/kylinolap/job/flow/JobFlow.java +++ b/job/src/main/java/com/kylinolap/job/flow/JobFlow.java @@ -153,6 +153,7 @@ private String wrapExecCmd(JobInstance job, String cmd, String suffix) { } String log = config.getKylinJobLogDir() + "/" + job.getUuid() + "_" + suffix + ".log"; - return "set -o pipefail; " + cmd + " 2>&1 | tee " + log; + String mkLogDir = "mkdir -p " + config.getKylinJobLogDir(); + return mkLogDir + ";" + "set -o pipefail; " + cmd + " 2>&1 | tee " + log; } } From adabbcaa988f23a3e909b2f07513b2b13fdead6e Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Wed, 19 Nov 2014 18:05:15 +0800 Subject: [PATCH 03/24] refactor --- cube/src/main/java/com/kylinolap/cube/CubeManager.java | 2 +- .../java/com/kylinolap/cube/CubeSegmentValidator.java | 17 +++++++---------- .../main/java/com/kylinolap/job/JobInstanceBuilder.java | 13 +++++++++---- .../kylinolap/job/hadoop/cube/MergeCuboidMapper.java | 5 +++-- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java index 1c01dab..3578c27 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java @@ -428,7 +428,7 @@ public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc d * For the new segment, we need to create dictionaries for it, too. For * those dictionaries on fact table, create it by merging underlying * dictionaries For those dictionaries on lookup table, just copy it from - * any one of the merging segments, it's ganranteed to be consistent(checked + * any one of the merging segments, it's guaranteed to be consistent(checked * in CubeSegmentValidator) * * @param cube diff --git a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java index 524734b..d2e69c9 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java @@ -38,13 +38,13 @@ private CubeSegmentValidator() { public static CubeSegmentValidator getCubeSegmentValidator(CubeBuildTypeEnum buildType, CubePartitionType partitionType) { switch (buildType) { case MERGE: - return new CubeSegmentValidator().new MergeOperationValidator(); + return new MergeOperationValidator(); case BUILD: switch (partitionType) { case APPEND: - return new CubeSegmentValidator().new IncrementalBuildOperationValidator(); + return new IncrementalBuildOperationValidator(); case UPDATE_INSERT: - return new CubeSegmentValidator().new UpdateBuildOperationValidator(); + return new UpdateBuildOperationValidator(); } default: return new CubeSegmentValidator(); @@ -54,7 +54,7 @@ public static CubeSegmentValidator getCubeSegmentValidator(CubeBuildTypeEnum bui void validate(CubeInstance cubeInstance, List newSegments) throws CubeIntegrityException { } - public class MergeOperationValidator extends CubeSegmentValidator { + public static class MergeOperationValidator extends CubeSegmentValidator { private void checkContingency(CubeInstance cubeInstance, List newSegments) throws CubeIntegrityException { if (cubeInstance.getSegments().size() < 2) { throw new CubeIntegrityException("No segments to merge."); @@ -141,7 +141,7 @@ public void validate(CubeInstance cubeInstance, List newSegments) t } } - public class IncrementalBuildOperationValidator extends CubeSegmentValidator { + public static class IncrementalBuildOperationValidator extends CubeSegmentValidator { /* * (non-Javadoc) * @@ -151,10 +151,6 @@ public void validate(CubeInstance cubeInstance, List newSegments) t */ @Override void validate(CubeInstance cubeInstance, List newSegments) throws CubeIntegrityException { - this.checkContingency(cubeInstance, newSegments); - } - - void checkContingency(CubeInstance cubeInstance, List newSegments) throws CubeIntegrityException { if (newSegments.size() != 1) { throw new CubeIntegrityException("Invalid date range."); } @@ -185,9 +181,10 @@ void checkContingency(CubeInstance cubeInstance, List newSegments) } } } + } - public class UpdateBuildOperationValidator extends CubeSegmentValidator { + public static class UpdateBuildOperationValidator extends CubeSegmentValidator { /* * (non-Javadoc) diff --git a/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java b/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java index bb73f1c..e54b244 100644 --- a/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java +++ b/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java @@ -145,14 +145,15 @@ private String getRowkeyDistributionOutputPath() { private List createMergeCubeSegmentsSteps(JobInstance jobInstance) throws IOException { - if (cube.getMergingSegments() == null || cube.getMergingSegments().size() < 2) { + List mergingSegments = cube.getMergingSegments(); + if (mergingSegments == null || mergingSegments.size() < 2) { throw new IllegalArgumentException("Merging segments count should be more than 2"); } - String[] cuboidPaths = new String[cube.getMergingSegments().size()]; - for (int i = 0; i < cube.getMergingSegments().size(); i++) { - CubeSegment seg = cube.getMergingSegments().get(i); + String[] cuboidPaths = new String[mergingSegments.size()]; + for (int i = 0; i < mergingSegments.size(); i++) { + CubeSegment seg = mergingSegments.get(i); cuboidPaths[i] = JobInstance.getJobWorkingDir(seg.getLastBuildJobID(), engineConfig.getHdfsWorkingDirectory()) + "/" + jobInstance.getRelatedCube() + "/cuboid/*"; } String formattedPath = formatPaths(cuboidPaths); @@ -241,6 +242,10 @@ private String formatPaths(String[] paths) { return StringUtils.join(paths, ","); } + private String formatPaths(List paths) { + return StringUtils.join(paths, ","); + } + private JobStep createBuildDictionaryStep(JobInstance jobInstance, int stepSeqNum) { // base cuboid job JobStep buildDictionaryStep = new JobStep(); diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java index 10c6456..5f5dd81 100644 --- a/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java +++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java @@ -65,6 +65,8 @@ private HashMap dictsNeedMerging = new HashMap(); + private static final Pattern JOB_NAME_PATTERN = Pattern.compile("kylin-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"); + private Boolean checkNeedMerging(TblColRef col) throws IOException { Boolean ret = dictsNeedMerging.get(col); if (ret != null) @@ -77,8 +79,7 @@ private Boolean checkNeedMerging(TblColRef col) throws IOException { } private String extractJobIDFromPath(String path) { - Pattern pattern = Pattern.compile("kylin-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"); - Matcher matcher = pattern.matcher(path); + Matcher matcher = JOB_NAME_PATTERN.matcher(path); // check the first occurance if (matcher.find()) { return matcher.group(1); From 7b4f4f22267a716fceecc9b8c204b6e21d7a2de0 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Wed, 19 Nov 2014 20:38:07 +0800 Subject: [PATCH 04/24] refactor --- .../java/com/kylinolap/job/JobInstanceBuilder.java | 33 ++++++++++++---------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java b/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java index e54b244..1cd9195 100644 --- a/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java +++ b/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java @@ -122,14 +122,14 @@ private String getIntermediateHiveTablePath() { return JoinedFlatTable.getTableDir(intermediateTableDesc, jobWorkingDir, jobUUID); } - private String[] getCuboidOutputPaths(String cubeName, int totalRowkeyColumnCount, int groupRowkeyColumnsCount) { + private String[] getCuboidOutputPaths(String cuboidRootPath, int totalRowkeyColumnCount, int groupRowkeyColumnsCount) { String[] paths = new String[groupRowkeyColumnsCount + 1]; for (int i = 0; i <= groupRowkeyColumnsCount; i++) { int dimNum = totalRowkeyColumnCount - i; if (dimNum == totalRowkeyColumnCount) { - paths[i] = jobWorkingDir + "/" + cubeName + "/cuboid/" + "base_cuboid"; + paths[i] = cuboidRootPath + "base_cuboid"; } else { - paths[i] = jobWorkingDir + "/" + cubeName + "/cuboid/" + dimNum + "d_cuboid"; + paths[i] = cuboidRootPath + dimNum + "d_cuboid"; } } return paths; @@ -162,18 +162,19 @@ private String getRowkeyDistributionOutputPath() { // jobInstance.clearSteps(); int stepSeqNum = 0; List result = Lists.newArrayList(); + final String mergedCuboidPath = jobWorkingDir + "/" + cubeName + "/merged_cuboid"; // merge cuboid data of ancestor segments - result.add(createMergeCuboidDataStep(jobInstance, stepSeqNum++, formattedPath)); + result.add(createMergeCuboidDataStep(jobInstance, stepSeqNum++, formattedPath, mergedCuboidPath)); // get output distribution step - result.add(createRangeRowkeyDistributionStep(jobInstance, stepSeqNum++, jobWorkingDir + "/" + cubeName + "/merged_cuboid")); + result.add(createRangeRowkeyDistributionStep(jobInstance, stepSeqNum++, mergedCuboidPath)); // create htable step result.add(createCreateHTableStep(jobInstance, stepSeqNum++)); // generate hfiles step - result.add(createConvertCuboidToHfileStep(jobInstance, stepSeqNum++, jobWorkingDir + "/" + cubeName + "/merged_cuboid")); + result.add(createConvertCuboidToHfileStep(jobInstance, stepSeqNum++, mergedCuboidPath)); // bulk load step result.add(createBulkLoadStep(jobInstance, stepSeqNum++)); @@ -194,7 +195,6 @@ private String getRowkeyDistributionOutputPath() { int groupRowkeyColumnsCount = cube.getDescriptor().getRowkey().getNCuboidBuildLevels(); int totalRowkeyColumnsCount = cube.getDescriptor().getRowkey().getRowKeyColumns().length; - String[] cuboidOutputTempPath = getCuboidOutputPaths(cubeName, totalRowkeyColumnsCount, groupRowkeyColumnsCount); int stepSeqNum = 0; List result = Lists.newArrayList(); @@ -202,15 +202,18 @@ private String getRowkeyDistributionOutputPath() { // by default in here // flat hive table step - result.add(createIntermediateHiveTableStep(jobInstance, stepSeqNum++, cuboidOutputTempPath)); + result.add(createIntermediateHiveTableStep(jobInstance, stepSeqNum++)); } // fact distinct columns step - result.add(createFactDistinctColumnsStep(jobInstance, stepSeqNum++, cuboidOutputTempPath)); + result.add(createFactDistinctColumnsStep(jobInstance, stepSeqNum++)); // build dictionary step result.add(createBuildDictionaryStep(jobInstance, stepSeqNum++)); + final String cuboidRootPath = jobWorkingDir + "/" + cubeName + "/cuboid/"; + + String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount); // base cuboid step result.add(createBaseCuboidStep(jobInstance, stepSeqNum++, cuboidOutputTempPath)); @@ -221,12 +224,12 @@ private String getRowkeyDistributionOutputPath() { } // get output distribution step - result.add(createRangeRowkeyDistributionStep(jobInstance, stepSeqNum++, jobWorkingDir + "/" + cubeName + "/cuboid/*")); + result.add(createRangeRowkeyDistributionStep(jobInstance, stepSeqNum++, cuboidRootPath + "*")); // create htable step result.add(createCreateHTableStep(jobInstance, stepSeqNum++)); // generate hfiles step - result.add(createConvertCuboidToHfileStep(jobInstance, stepSeqNum++, jobWorkingDir + "/" + cubeName + "/cuboid/*")); + result.add(createConvertCuboidToHfileStep(jobInstance, stepSeqNum++, cuboidRootPath + "*")); // bulk load step result.add(createBulkLoadStep(jobInstance, stepSeqNum++)); @@ -263,7 +266,7 @@ private JobStep createBuildDictionaryStep(JobInstance jobInstance, int stepSeqNu return buildDictionaryStep; } - private JobStep createIntermediateHiveTableStep(JobInstance jobInstance, int stepSeqNum, String[] cuboidOutputTempPath) throws IOException { + private JobStep createIntermediateHiveTableStep(JobInstance jobInstance, int stepSeqNum) throws IOException { JoinedFlatTableDesc intermediateTableDesc = new JoinedFlatTableDesc(cube.getDescriptor(), this.cubeSegment); String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, jobUUID); String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobWorkingDir, jobUUID); @@ -287,7 +290,7 @@ private JobStep createIntermediateHiveTableStep(JobInstance jobInstance, int ste return intermediateHiveTableStep; } - private JobStep createFactDistinctColumnsStep(JobInstance jobInstance, int stepSeqNum, String[] cuboidOutputTempPath) throws IOException { + private JobStep createFactDistinctColumnsStep(JobInstance jobInstance, int stepSeqNum) throws IOException { // base cuboid job JobStep factDistinctColumnsStep = new JobStep(); @@ -388,7 +391,7 @@ private JobStep createRangeRowkeyDistributionStep(JobInstance jobInstance, int s return rowkeyDistributionStep; } - private JobStep createMergeCuboidDataStep(JobInstance jobInstance, int stepSeqNum, String inputPath) throws IOException { + private JobStep createMergeCuboidDataStep(JobInstance jobInstance, int stepSeqNum, String inputPath, String outputPath) throws IOException { JobStep mergeCuboidDataStep = new JobStep(); mergeCuboidDataStep.setName(JobConstants.STEP_NAME_MERGE_CUBOID); String cmd = ""; @@ -397,7 +400,7 @@ private JobStep createMergeCuboidDataStep(JobInstance jobInstance, int stepSeqNu cmd = appendExecCmdParameters(cmd, "cubename", cubeName); cmd = appendExecCmdParameters(cmd, "segmentname", segmentName); cmd = appendExecCmdParameters(cmd, "input", inputPath); - cmd = appendExecCmdParameters(cmd, "output", jobWorkingDir + "/" + cubeName + "/merged_cuboid"); + cmd = appendExecCmdParameters(cmd, "output", outputPath); cmd = appendExecCmdParameters(cmd, "jobname", "Kylin_Merge_Cuboid_" + jobInstance.getRelatedCube() + "_Step_" + stepSeqNum); mergeCuboidDataStep.setExecCmd(cmd); From e1a96c0be23617ccada20daa1645b57626d399a7 Mon Sep 17 00:00:00 2001 From: honma Date: Thu, 20 Nov 2014 10:00:14 +0800 Subject: [PATCH 05/24] fix #76 --- .../java/com/kylinolap/cube/dataGen/FactTableGenerator.java | 2 +- .../com/kylinolap/job/hadoop/cube/NewBaseCuboidMapper.java | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/dataGen/FactTableGenerator.java b/cube/src/main/java/com/kylinolap/cube/dataGen/FactTableGenerator.java index 8ccca7c..415353a 100644 --- a/cube/src/main/java/com/kylinolap/cube/dataGen/FactTableGenerator.java +++ b/cube/src/main/java/com/kylinolap/cube/dataGen/FactTableGenerator.java @@ -220,7 +220,7 @@ private void prepare() throws Exception { public int compare(DimensionDesc o1, DimensionDesc o2) { JoinDesc j1 = o2.getJoin(); JoinDesc j2 = o1.getJoin(); - return Integer.compare(j1 != null ? j1.getPrimaryKey().length : 0, j2 != null ? j2.getPrimaryKey().length : 0); + return Integer.valueOf(j1 != null ? j1.getPrimaryKey().length : 0).compareTo(j2 != null ? j2.getPrimaryKey().length : 0); } }); return dimensions; diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/NewBaseCuboidMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/NewBaseCuboidMapper.java index 66beac4..03ea2a0 100644 --- a/job/src/main/java/com/kylinolap/job/hadoop/cube/NewBaseCuboidMapper.java +++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/NewBaseCuboidMapper.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; -import org.eclipse.jdt.internal.core.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,13 +93,13 @@ private HashMap lookupTables;// name -> table private LinkedList tableJoins; private LinkedList> factTblColAsRowKey;// similar as - // TableJoin.dimTblColAsRowKey + // TableJoin.dimTblColAsRowKey private int[][] measureColumnIndice; private byte[] nullValue; private class TableJoin { public LinkedList fkIndice;// zero-based join columns on fact - // table + // table public String lookupTableName; public String joinType; @@ -146,8 +145,8 @@ protected void setup(Context context) throws IOException { bytesSplitter = new BytesSplitter(factTableDesc.getColumns().length, 4096); nullValue = new byte[] { (byte) '\\', (byte) 'N' };// As in Hive, null - // value is - // represented by \N + // value is + // represented by \N prepareJoins(); prepareMetrics(); @@ -264,7 +263,7 @@ private void prepareMetrics() { filledDimension++; } - Assert.isTrue(filledDimension == keyBytesBuf.length); + assert filledDimension == keyBytesBuf.length; // all the row key slots(keyBytesBuf) should be complete now return rowKeyEncoder.encode(keyBytesBuf); From 5bc2f45b707b9753397214f71338ee5a34e4d0dd Mon Sep 17 00:00:00 2001 From: honma Date: Thu, 20 Nov 2014 15:07:45 +0800 Subject: [PATCH 06/24] change demo tomcat's port to 7070, and remove the username:password in kylin.proerties --- .../kylinolap/common/restclient/RestClient.java | 25 +++------------------- .../common/restclient/RestClientTest.java | 24 +++++++++++++++++++++ deploy.sh | 4 ++-- deploy/server.xml | 2 +- examples/test_case_data/kylin.properties | 2 +- .../com/kylinolap/job/CubeDevelopTestCase.java | 1 + .../main/java/com/kylinolap/rest/DebugTomcat.java | 2 +- .../com/kylinolap/rest/service/JobService.java | 5 ++--- server/src/main/resources/kylinSecurity.xml | 1 + .../rest/controller/CubeControllerTest.java | 8 +++++-- 10 files changed, 42 insertions(+), 32 deletions(-) create mode 100644 common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java diff --git a/common/src/main/java/com/kylinolap/common/restclient/RestClient.java b/common/src/main/java/com/kylinolap/common/restclient/RestClient.java index 024cc63..5f1367f 100644 --- a/common/src/main/java/com/kylinolap/common/restclient/RestClient.java +++ b/common/src/main/java/com/kylinolap/common/restclient/RestClient.java @@ -44,7 +44,7 @@ String password; HttpClient client; - private static Pattern fullRestPattern = Pattern.compile("([^:]+)[:]([^@]+)[@]([^:]+)(?:[:](\\d+))?"); + private static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?"); public static boolean matchFullRestPattern(String uri) { Matcher m = fullRestPattern.matcher(uri); @@ -57,37 +57,18 @@ public static boolean matchFullRestPattern(String uri) { */ public RestClient(String uri) { Matcher m = fullRestPattern.matcher(uri); - if (m.matches() == false) + if (!m.matches()) throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + fullRestPattern); String user = m.group(1); String pwd = m.group(2); String host = m.group(3); String portStr = m.group(4); - int port = Integer.parseInt(portStr == null ? "80" : portStr); + int port = Integer.parseInt(portStr == null ? "7070" : portStr); init(host, port, user, pwd); } - public RestClient(String hostAndPort, String userName, String password) { - String host; - int port; - int cut = hostAndPort.indexOf(":"); - if (cut < 0) { - host = hostAndPort; - port = 80; - } else { - host = hostAndPort.substring(0, cut); - port = Integer.parseInt(hostAndPort.substring(cut + 1)); - } - - init(host, port, userName, password); - } - - public RestClient(String host, int port, String userName, String password) { - init(host, port, userName, password); - } - private void init(String host, int port, String userName, String password) { this.host = host; this.port = port; diff --git a/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java b/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java new file mode 100644 index 0000000..2b35ef9 --- /dev/null +++ b/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java @@ -0,0 +1,24 @@ +package com.kylinolap.common.restclient; + + +import org.junit.Test; + +import java.io.IOException; + +public class RestClientTest { + + @Test + public void basicTests() throws IOException { + RestClient a = new RestClient("prod01:80"); + //a.wipeCache("metadata", "a", "a"); + //String aa = a.getKylinProperties(); + //System.out.println(aa); + RestClient b = new RestClient("sandbox.hortonworks.com:9080"); + //b.wipeCache("metadata", "a", "a"); + //String bb = b.getKylinProperties(); + //System.out.println(bb); + + + } + +} \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index c7432c9..d810616 100755 --- a/deploy.sh +++ b/deploy.sh @@ -19,7 +19,7 @@ echo "This script will help you:" echo "1. Check environment" echo "2. Build Kylin artifacts" echo "3. Prepare test cube related data" -echo "4. Lauch a web service to build cube and query with (at http://localhost:9080)" +echo "4. Lauch a web service to build cube and query with (at http://localhost:7070)" echo "Please make sure you are running this script on a hadoop CLI machine, and you have enough permissions." echo "Also, We assume you have installed: JAVA, TOMCAT, NPM and MAVEN." echo "[Warning] The installation may break existing tomcat applications on this CLI" @@ -203,4 +203,4 @@ sudo -i "${CATALINA_HOME}/bin/startup.sh" echo "Kylin-Deploy Success!" -echo "Please visit http://:9080 to play with the cubes! (Useranme: ADMIN, Password: KYLIN)" +echo "Please visit http://:7070 to play with the cubes! (Useranme: ADMIN, Password: KYLIN)" diff --git a/deploy/server.xml b/deploy/server.xml index 8c69845..4bd26d0 100644 --- a/deploy/server.xml +++ b/deploy/server.xml @@ -67,7 +67,7 @@ APR (HTTP/AJP) Connector: /docs/apr.html Define a non-SSL HTTP/1.1 Connector on port 8080 --> - = 1) { System.setProperty("kylin.metadata.url", args[0]); } - int port = 80; + int port = 7070; if (args.length >= 2) { port = Integer.parseInt(args[1]); } diff --git a/server/src/main/java/com/kylinolap/rest/service/JobService.java b/server/src/main/java/com/kylinolap/rest/service/JobService.java index d5c9ed8..815f480 100644 --- a/server/src/main/java/com/kylinolap/rest/service/JobService.java +++ b/server/src/main/java/com/kylinolap/rest/service/JobService.java @@ -43,7 +43,6 @@ /** * @author ysong1 - * */ @Component("jobService") public class JobService extends BasicService { @@ -58,7 +57,7 @@ Integer offset = (null == offsetValue) ? 0 : offsetValue; List jobs = listAllJobs(cubeName, projectName, statusList); Collections.sort(jobs); - + if (jobs.size() <= offset) { return Collections.emptyList(); } @@ -101,7 +100,7 @@ public String submitJob(CubeInstance cube, long startDate, long endDate, CubeBui List jobInstances = this.getJobManager().listJobs(cube.getName(), null); for (JobInstance jobInstance : jobInstances) { if (jobInstance.getStatus() == JobStatusEnum.PENDING || jobInstance.getStatus() == JobStatusEnum.RUNNING) { - throw new JobException("The cube " + cube.getName() + " has running job, please discard it and try again."); + throw new JobException("The cube " + cube.getName() + " has running job(" + jobInstance.getUuid() + ") please discard it and try again."); } } diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml index 9b1b1fd..38ab9f9 100644 --- a/server/src/main/resources/kylinSecurity.xml +++ b/server/src/main/resources/kylinSecurity.xml @@ -16,6 +16,7 @@ + diff --git a/server/src/test/java/com/kylinolap/rest/controller/CubeControllerTest.java b/server/src/test/java/com/kylinolap/rest/controller/CubeControllerTest.java index 6a17ee9..5fd0a08 100644 --- a/server/src/test/java/com/kylinolap/rest/controller/CubeControllerTest.java +++ b/server/src/test/java/com/kylinolap/rest/controller/CubeControllerTest.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.StringWriter; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -33,7 +34,6 @@ /** * @author xduo - * */ public class CubeControllerTest extends TestBase { @@ -65,7 +65,8 @@ public void testBasics() throws IOException { CubeDesc cube = cubes[0]; CubeDesc newCube = new CubeDesc(); - newCube.setName(cube.getName() + "_test_save"); + String newCubeName = cube.getName() + "_test_save"; + newCube.setName(newCubeName); newCube.setDimensions(cube.getDimensions()); newCube.setHBaseMapping(cube.getHBaseMapping()); newCube.setMeasures(cube.getMeasures()); @@ -80,5 +81,8 @@ public void testBasics() throws IOException { CubeRequest cubeRequest = new CubeRequest(); cubeRequest.setCubeDescData(stringWriter.toString()); cubeRequest = cubeController.saveCubeDesc(cubeRequest); + + cubeController.deleteCube(newCubeName); } + } From 415b803eeb8c1c190958be7efecbaccf79675f3b Mon Sep 17 00:00:00 2001 From: honma Date: Thu, 20 Nov 2014 16:13:31 +0800 Subject: [PATCH 07/24] conditional save project instance when project listing table --- .../com/kylinolap/cube/project/ProjectManager.java | 41 ++++++++++++---------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java b/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java index 81b9f68..7113753 100644 --- a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java +++ b/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java @@ -51,7 +51,6 @@ /** * @author xduo - * */ public class ProjectManager { private static final Logger logger = LoggerFactory.getLogger(ProjectManager.class); @@ -64,7 +63,7 @@ // project name => ProjrectDesc private SingleValueCache projectMap = new SingleValueCache(Broadcaster.TYPE.PROJECT); // project name => tables - private Multimap projectTables = Multimaps.synchronizedMultimap(HashMultimap. create()); + private Multimap projectTables = Multimaps.synchronizedMultimap(HashMultimap.create()); public static ProjectManager getInstance(KylinConfig config) { ProjectManager r = CACHE.get(config); @@ -189,8 +188,8 @@ public ProjectInstance updateCubeToProject(String cubeName, String newProjectNam return addCubeToProject(cubeName, newProjectName, owner); } - - public ProjectInstance updateTableToProject(String tables,String projectName) throws IOException { + + public ProjectInstance updateTableToProject(String tables, String projectName) throws IOException { ProjectInstance projectInstance = getProject(projectName); String[] tokens = StringUtils.split(tables, ","); for (int i = 0; i < tokens.length; i++) { @@ -199,17 +198,17 @@ public ProjectInstance updateTableToProject(String tables,String projectName) th projectInstance.addTable(token); } } - + List exposedTables = listExposedTables(projectName); - for(TableDesc table : exposedTables){ + for (TableDesc table : exposedTables) { projectInstance.addTable(table.getName()); } - + saveResource(projectInstance); return projectInstance; } - - + + public void removeCubeFromProjects(String cubeName) throws IOException { for (ProjectInstance projectInstance : findProjects(cubeName)) { projectInstance.removeCube(cubeName); @@ -232,19 +231,23 @@ public void removeCubeFromProjects(String cubeName) throws IOException { return tables; } - - + public List listDefinedTablesInProject(String project) throws IOException { project = ProjectInstance.getNormalizedProjectName(project); ProjectInstance projectInstance = getProject(project); - + + int originTableCount = projectInstance.getTablesCount(); //sync exposed table to project when list List exposedTables = listExposedTables(project); - for(TableDesc table : exposedTables){ + for (TableDesc table : exposedTables) { projectInstance.addTable(table.getName()); } - saveResource(projectInstance); - + + //only save project json if new tables are sync in + if (originTableCount < projectInstance.getTablesCount()) { + saveResource(projectInstance); + } + List tables = Lists.newArrayList(); for (String table : projectInstance.getTables()) { TableDesc tableDesc = getMetadataManager().getTableDesc(table); @@ -254,8 +257,8 @@ public void removeCubeFromProjects(String cubeName) throws IOException { } return tables; - } - + } + public List listExposedColumns(String project, String table) { project = ProjectInstance.getNormalizedProjectName(project); @@ -301,7 +304,7 @@ public boolean isExposedColumn(String project, String table, String col) { return new ArrayList(ret); } - + public List getCubesByTable(String project, String tableName) { project = ProjectInstance.getNormalizedProjectName(project); @@ -468,7 +471,7 @@ private ProjectInstance addCubeToProject(String cubeName, String project, String return newProject; } - + private void saveResource(ProjectInstance proj) throws IOException { ResourceStore store = getStore(); store.putResource(proj.getResourcePath(), proj, PROJECT_SERIALIZER); From ba730ba8290ea56b81a028b1dd92c54f7c4cdb18 Mon Sep 17 00:00:00 2001 From: honma Date: Thu, 20 Nov 2014 16:25:40 +0800 Subject: [PATCH 08/24] test case passed --- .../src/test/java/com/kylinolap/common/restclient/RestClientTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java b/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java index 2b35ef9..83cc07b 100644 --- a/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java +++ b/common/src/test/java/com/kylinolap/common/restclient/RestClientTest.java @@ -13,7 +13,7 @@ public void basicTests() throws IOException { //a.wipeCache("metadata", "a", "a"); //String aa = a.getKylinProperties(); //System.out.println(aa); - RestClient b = new RestClient("sandbox.hortonworks.com:9080"); + RestClient b = new RestClient("sandbox.hortonworks.com:7070"); //b.wipeCache("metadata", "a", "a"); //String bb = b.getKylinProperties(); //System.out.println(bb); From 7316bf083012d31de1c6e8803379f20658d6582e Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 20 Nov 2014 18:39:38 +0800 Subject: [PATCH 09/24] [maven-release-plugin] prepare release kylin-0.6.2 --- common/pom.xml | 168 ++++---- cube/pom.xml | 314 +++++++-------- dictionary/pom.xml | 146 +++---- jdbc/pom.xml | 160 ++++---- job/pom.xml | 392 +++++++++---------- metadata/pom.xml | 216 +++++------ pom.xml | 1074 ++++++++++++++++++++++++++-------------------------- query/pom.xml | 372 +++++++++--------- server/pom.xml | 780 +++++++++++++++++++------------------- storage/pom.xml | 200 +++++----- 10 files changed, 1912 insertions(+), 1910 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index b77354f..baaf4c8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -1,84 +1,84 @@ - - 4.0.0 - - kylin-common - jar - Kylin:Common - - - com.kylinolap - kylin - 0.6.2-SNAPSHOT - - - - - - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.commons - commons-lang3 - - - commons-io - commons-io - - - commons-configuration - commons-configuration - - - com.google.guava - guava - - - com.jcraft - jsch - - - com.ning - compress-lzf - - - - - - org.apache.hadoop - hadoop-common - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - - com.google.protobuf - protobuf-java - - - - - org.apache.hbase - hbase-common - provided - - - org.apache.hbase - hbase-client - provided - - - junit - junit - test - - - - - + + 4.0.0 + + kylin-common + jar + Kylin:Common + + + com.kylinolap + kylin + 0.6.2 + + + + + + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.commons + commons-lang3 + + + commons-io + commons-io + + + commons-configuration + commons-configuration + + + com.google.guava + guava + + + com.jcraft + jsch + + + com.ning + compress-lzf + + + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + + com.google.protobuf + protobuf-java + + + + + org.apache.hbase + hbase-common + provided + + + org.apache.hbase + hbase-client + provided + + + junit + junit + test + + + + + diff --git a/cube/pom.xml b/cube/pom.xml index 89a7942..173febd 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -1,157 +1,157 @@ - - 4.0.0 - - kylin-cube - jar - Kylin:Cube - - com.kylinolap - kylin - 0.6.2-SNAPSHOT - - - - - - com.kylinolap - kylin-metadata - ${project.parent.version} - - - com.kylinolap - kylin-dictionary - ${project.parent.version} - - - - commons-cli - commons-cli - - - commons-lang - commons-lang - - - commons-io - commons-io - - - commons-configuration - commons-configuration - - - log4j - log4j - - - com.fasterxml.jackson.core - jackson-databind - - - commons-httpclient - commons-httpclient - - - com.google.guava - guava - - - com.ning - compress-lzf - - - com.n3twork.druid - extendedset - - - - - - org.apache.hadoop - hadoop-common - provided - - - org.apache.hadoop - hadoop-annotations - provided - - - org.apache.hadoop - hadoop-mapreduce-client-core - provided - - - org.apache.hadoop - hadoop-minicluster - test - - - org.apache.mrunit - mrunit - hadoop2 - test - - - org.apache.hbase - hbase-hadoop2-compat - provided - - - org.apache.hbase - hbase-common - provided - - - org.apache.hbase - hbase-client - provided - - - org.apache.hbase - hbase-server - provided - - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - - - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - provided - - - junit - junit - test - - - - - + + 4.0.0 + + kylin-cube + jar + Kylin:Cube + + com.kylinolap + kylin + 0.6.2 + + + + + + com.kylinolap + kylin-metadata + ${project.parent.version} + + + com.kylinolap + kylin-dictionary + ${project.parent.version} + + + + commons-cli + commons-cli + + + commons-lang + commons-lang + + + commons-io + commons-io + + + commons-configuration + commons-configuration + + + log4j + log4j + + + com.fasterxml.jackson.core + jackson-databind + + + commons-httpclient + commons-httpclient + + + com.google.guava + guava + + + com.ning + compress-lzf + + + com.n3twork.druid + extendedset + + + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-annotations + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + org.apache.hadoop + hadoop-minicluster + test + + + org.apache.mrunit + mrunit + hadoop2 + test + + + org.apache.hbase + hbase-hadoop2-compat + provided + + + org.apache.hbase + hbase-common + provided + + + org.apache.hbase + hbase-client + provided + + + org.apache.hbase + hbase-server + provided + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + provided + + + junit + junit + test + + + + + diff --git a/dictionary/pom.xml b/dictionary/pom.xml index 6b21a78..2cdf37f 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -1,73 +1,73 @@ - - 4.0.0 - - kylin-dictionary - jar - Kylin:Dictionary - - - com.kylinolap - kylin - 0.6.2-SNAPSHOT - - - - - - - - - com.kylinolap - kylin-metadata - ${project.parent.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - commons-io - commons-io - - - commons-configuration - commons-configuration - - - com.google.guava - guava - - - - - - org.apache.hadoop - hadoop-common - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - - com.google.protobuf - protobuf-java - - - - - org.apache.hbase - hbase-common - provided - - - junit - junit - test - - - - + + 4.0.0 + + kylin-dictionary + jar + Kylin:Dictionary + + + com.kylinolap + kylin + 0.6.2 + + + + + + + + + com.kylinolap + kylin-metadata + ${project.parent.version} + + + + com.fasterxml.jackson.core + jackson-databind + + + commons-io + commons-io + + + commons-configuration + commons-configuration + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + + com.google.protobuf + protobuf-java + + + + + org.apache.hbase + hbase-common + provided + + + junit + junit + test + + + + diff --git a/jdbc/pom.xml b/jdbc/pom.xml index dbfce24..593553d 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -1,80 +1,80 @@ - - 4.0.0 - kylin-jdbc - jar - - - Kylin:Jdbc - kylin jdbc on optiq avatica - https://github.scm.corp.ebay.com/BIPlatform/Kylin - - - com.kylinolap - kylin - 0.6.2-SNAPSHOT - - - - UTF-8 - - - - - org.apache.calcite - calcite-core - - - org.apache.calcite - calcite-avatica - - - commons-httpclient - commons-httpclient - - - log4j - log4j - - - org.slf4j - jcl-over-slf4j - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - junit - junit - test - - - - - - - maven-assembly-plugin - - - jar-with-dependencies - - false - - - - make-assembly - package - - single - - - - - - - - + + 4.0.0 + kylin-jdbc + jar + + + Kylin:Jdbc + kylin jdbc on optiq avatica + https://github.scm.corp.ebay.com/BIPlatform/Kylin + + + com.kylinolap + kylin + 0.6.2 + + + + UTF-8 + + + + + org.apache.calcite + calcite-core + + + org.apache.calcite + calcite-avatica + + + commons-httpclient + commons-httpclient + + + log4j + log4j + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + junit + junit + test + + + + + + + maven-assembly-plugin + + + jar-with-dependencies + + false + + + + make-assembly + package + + single + + + + + + + + diff --git a/job/pom.xml b/job/pom.xml index 90eb233..525fc8d 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -1,196 +1,196 @@ - - - 4.0.0 - - com.kylinolap - kylin - 0.6.2-SNAPSHOT - - - kylin-job - Kylin:Job - http://maven.apache.org - - UTF-8 - - - - - - com.kylinolap - kylin-cube - ${project.parent.version} - - - - commons-cli - commons-cli - - - commons-lang - commons-lang - - - commons-io - commons-io - - - commons-configuration - commons-configuration - - - log4j - log4j - - - com.fasterxml.jackson.core - jackson-databind - - - commons-httpclient - commons-httpclient - - - com.google.guava - guava - - - - org.quartz-scheduler - quartz - - - org.quartz-scheduler - quartz-jobs - - - - commons-daemon - commons-daemon - - - - org.apache.curator - curator-framework - - - org.apache.zookeeper - zookeeper - - - - - org.apache.curator - curator-recipes - - - org.apache.zookeeper - zookeeper - - - - - org.apache.commons - commons-email - 1.1 - - - org.apache.hbase - hbase-common - - - - - - - org.apache.hadoop - hadoop-common - provided - - - org.apache.hadoop - hadoop-annotations - provided - - - org.apache.hadoop - hadoop-mapreduce-client-core - provided - - - org.apache.hadoop - hadoop-minicluster - test - - - org.apache.mrunit - mrunit - hadoop2 - test - - - org.apache.hbase - hbase-hadoop2-compat - provided - - - org.apache.hbase - hbase-client - provided - - - org.apache.hbase - hbase-server - provided - - - org.apache.hadoop - hadoop-yarn-server-resourcemanager - provided - - - junit - junit - test - - - org.apache.maven - maven-model - test - - - org.apache.hadoop - hadoop-yarn-api - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - - - - - - maven-assembly-plugin - - - create-job-jar - package - - single - - - - src/main/assembly/job.xml - - - - - - - - + + + 4.0.0 + + com.kylinolap + kylin + 0.6.2 + + + kylin-job + Kylin:Job + http://maven.apache.org + + UTF-8 + + + + + + com.kylinolap + kylin-cube + ${project.parent.version} + + + + commons-cli + commons-cli + + + commons-lang + commons-lang + + + commons-io + commons-io + + + commons-configuration + commons-configuration + + + log4j + log4j + + + com.fasterxml.jackson.core + jackson-databind + + + commons-httpclient + commons-httpclient + + + com.google.guava + guava + + + + org.quartz-scheduler + quartz + + + org.quartz-scheduler + quartz-jobs + + + + commons-daemon + commons-daemon + + + + org.apache.curator + curator-framework + + + org.apache.zookeeper + zookeeper + + + + + org.apache.curator + curator-recipes + + + org.apache.zookeeper + zookeeper + + + + + org.apache.commons + commons-email + 1.1 + + + org.apache.hbase + hbase-common + + + + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-annotations + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + org.apache.hadoop + hadoop-minicluster + test + + + org.apache.mrunit + mrunit + hadoop2 + test + + + org.apache.hbase + hbase-hadoop2-compat + provided + + + org.apache.hbase + hbase-client + provided + + + org.apache.hbase + hbase-server + provided + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + provided + + + junit + junit + test + + + org.apache.maven + maven-model + test + + + org.apache.hadoop + hadoop-yarn-api + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + + + + + + maven-assembly-plugin + + + create-job-jar + package + + single + + + + src/main/assembly/job.xml + + + + + + + + diff --git a/metadata/pom.xml b/metadata/pom.xml index a4855b4..59015c3 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -1,108 +1,108 @@ - - 4.0.0 - - kylin-metadata - jar - Kylin:Metadata - - - com.kylinolap - kylin - 0.6.2-SNAPSHOT - - - - - - - - com.kylinolap - kylin-common - ${project.parent.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.commons - commons-lang3 - - - commons-io - commons-io - - - commons-configuration - commons-configuration - - - com.google.guava - guava - - - - - - org.apache.hadoop - hadoop-common - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - - com.google.protobuf - protobuf-java - - - - - org.apache.hbase - hbase-common - provided - - - org.apache.hbase - hbase-client - provided - - - junit - junit - test - - - - - + + 4.0.0 + + kylin-metadata + jar + Kylin:Metadata + + + com.kylinolap + kylin + 0.6.2 + + + + + + + + com.kylinolap + kylin-common + ${project.parent.version} + + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.commons + commons-lang3 + + + commons-io + commons-io + + + commons-configuration + commons-configuration + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + + com.google.protobuf + protobuf-java + + + + + org.apache.hbase + hbase-common + provided + + + org.apache.hbase + hbase-client + provided + + + junit + junit + test + + + + + diff --git a/pom.xml b/pom.xml index 5a4e5e8..31802e9 100644 --- a/pom.xml +++ b/pom.xml @@ -1,536 +1,538 @@ - - - 4.0.0 - com.kylinolap - kylin - pom - 0.6.2-SNAPSHOT - Kylin:HadoopOLAPEngine - - - - 1.7 - UTF-8 - UTF-8 - - - 2.4.1 - 2.4.1 - 0.98.0-hadoop2 - 3.4.5 - 0.13.0 - - - 3.4 - 4.11 - 1.0.0 - - 1.2 - 2.6 - 3.1 - 2.4 - 1.9 - 1.0.15 - - - 1.2.17 - 1.6.4 - 2.2.3 - 3.1 - 12.0.1 - - 0.1.51 - 2.5.0 - 3.1.1 - 1.3.174 - 2.9.1 - 2.7.1 - 0.8.4 - 1.3.4 - - - 3.1.2.RELEASE - - - 0.9.1-incubating - 0.4 - - - 3.0.1 - - - 2.2.1 - - - 2.6.0 - - - jacoco - reuseReports - ${project.basedir}/../target/jacoco.exec - java - com/kylinolap/**/tools/**:net/hydromatic/optiq/**:org/eigenbase/sql2rel/** - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop2.version} - compile - - - javax.servlet - servlet-api - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop2.version} - - - org.apache.hadoop - hadoop-mapreduce-client-app - ${hadoop2.version} - - - org.apache.hadoop - hadoop-yarn-api - ${hadoop2.version} - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop2.version} - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - ${hadoop2.version} - - - org.apache.hadoop - hadoop-annotations - ${hadoop2.version} - - - org.apache.hadoop - hadoop-auth - ${hadoop2.version} - - - org.apache.hadoop - hadoop-minicluster - ${hadoop2.version} - true - - - - - org.apache.hbase - hbase-hadoop2-compat - ${hbase-hadoop2.version} - - - org.apache.hbase - hbase-common - ${hbase-hadoop2.version} - - - org.apache.hbase - hbase-client - ${hbase-hadoop2.version} - - - org.apache.hbase - hbase-server - ${hbase-hadoop2.version} - - - org.apache.mrunit - mrunit - ${mrunit.version} - hadoop2 - - - - - org.apache.hive - hive-jdbc - ${hive.version} - - - - - org.apache.hadoop - hadoop-yarn-server-resourcemanager - ${yarn.version} - - - - - org.apache.calcite - calcite-core - ${optiq.version} - - - org.apache.calcite - calcite-avatica - ${optiq.version} - - - net.hydromatic - linq4j - ${linq4j.version} - - - - - junit - junit - ${junit.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - commons-cli - commons-cli - ${commons-cli.version} - - - commons-lang - commons-lang - ${commons-lang.version} - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - commons-io - commons-io - ${commons-io.version} - - - commons-configuration - commons-configuration - ${commons-configuration.version} - - - commons-daemon - commons-daemon - ${commons-daemon.version} - - - log4j - log4j - ${log4j.version} - - - org.slf4j - jcl-over-slf4j - ${slf4j.version} - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - com.google.guava - guava - ${guava.version} - - - com.jcraft - jsch - ${jsch.version} - - - org.dbunit - dbunit - ${dbunit.version} - - - org.apache.maven - maven-model - ${maven-model.version} - - - com.h2database - h2 - ${h2.version} - - - xerces - xercesImpl - ${xerces.version} - - - xalan - xalan - ${xalan.version} - - - com.ning - compress-lzf - ${compress-lzf.version} - - - com.n3twork.druid - extendedset - ${extendedset.version} - - - - org.quartz-scheduler - quartz - ${quartz.version} - - - org.quartz-scheduler - quartz-jobs - ${quartz.version} - - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - - - - - - central - http://repo.maven.apache.org/maven2 - - - conjars - http://conjars.org/repo/ - - - - true - always - warn - - - false - never - fail - - HDPReleases - HDP Releases - http://repo.hortonworks.com/content/repositories/releases/ - default - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - ${javaVersion} - ${javaVersion} - - - - org.apache.maven.plugins - maven-site-plugin - 2.0 - - - org.apache.maven.plugins - maven-install-plugin - 2.2 - - - org.apache.maven.plugins - maven-resources-plugin - 2.4 - - UTF-8 - - - - org.apache.maven.plugins - maven-jar-plugin - 2.4 - - - - test-jar - - - - - - org.apache.maven.plugins - maven-war-plugin - 2.0.2 - - - org.apache.maven.plugins - maven-dependency-plugin - - - org.apache.maven.plugins - maven-antrun-plugin - - - org.apache.maven.plugins - maven-source-plugin - 2.1.2 - - - attach-sources - package - - jar-no-fork - - - - - true - - - - org.antlr - antlr3-maven-plugin - ${antlr.version} - - - - - antlr - - false - - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.6 - - - - integration-test - verify - - - - - - org.apache.maven.plugins - maven-release-plugin - 2.1 - - - org.jacoco - jacoco-maven-plugin - 0.7.0.201403182114 - - ${sonar.jacoco.reportPath} - true - - - - agent - - prepare-agent - - - - - - - - - - scm:git:git@github.com:KylinOLAP/Kylin.git - scm:git:git@github.com:KylinOLAP/Kylin.git - scm:git:git@github.com:KylinOLAP/Kylin.git - - - - common - metadata - dictionary - cube - job - storage - query - server - jdbc - - - - default - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.16 - - - **/BuildCubeWithEngineTest.java - **/SampleCubeSetupTest.java - **/KylinQueryTest.java - - - - - - - - - e2e - - - + + + 4.0.0 + com.kylinolap + kylin + pom + 0.6.2 + Kylin:HadoopOLAPEngine + + + + 1.7 + 3.1.1 + UTF-8 + UTF-8 + + + 2.4.1 + 2.4.1 + 0.98.0-hadoop2 + 3.4.5 + 0.13.0 + + + 3.4 + + + 4.11 + 2.5.0 + 1.0.0 + 1.3.174 + + + 1.2 + 2.6 + 3.1 + 2.4 + 1.9 + 1.0.15 + 3.1 + + + 1.2.17 + 1.6.4 + 2.2.3 + 12.0.1 + 0.1.51 + 2.9.1 + 2.7.1 + 0.8.4 + 1.3.4 + + + 3.1.2.RELEASE + + + 0.9.1-incubating + 0.4 + + + 3.0.1 + + + 2.2.1 + + + 2.6.0 + + + jacoco + reuseReports + ${project.basedir}/../target/jacoco.exec + java + com/kylinolap/**/tools/**:net/hydromatic/optiq/**:org/eigenbase/sql2rel/** + + + + + + + org.apache.hadoop + hadoop-common + ${hadoop2.version} + compile + + + javax.servlet + servlet-api + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop2.version} + + + org.apache.hadoop + hadoop-mapreduce-client-app + ${hadoop2.version} + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop2.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop2.version} + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + ${hadoop2.version} + + + org.apache.hadoop + hadoop-annotations + ${hadoop2.version} + + + org.apache.hadoop + hadoop-auth + ${hadoop2.version} + + + org.apache.hadoop + hadoop-minicluster + ${hadoop2.version} + true + + + + + org.apache.hbase + hbase-hadoop2-compat + ${hbase-hadoop2.version} + + + org.apache.hbase + hbase-common + ${hbase-hadoop2.version} + + + org.apache.hbase + hbase-client + ${hbase-hadoop2.version} + + + org.apache.hbase + hbase-server + ${hbase-hadoop2.version} + + + org.apache.mrunit + mrunit + ${mrunit.version} + hadoop2 + + + + + org.apache.hive + hive-jdbc + ${hive.version} + + + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + ${yarn.version} + + + + + org.apache.calcite + calcite-core + ${optiq.version} + + + org.apache.calcite + calcite-avatica + ${optiq.version} + + + net.hydromatic + linq4j + ${linq4j.version} + + + + + junit + junit + ${junit.version} + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + commons-cli + commons-cli + ${commons-cli.version} + + + commons-lang + commons-lang + ${commons-lang.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + commons-io + commons-io + ${commons-io.version} + + + commons-configuration + commons-configuration + ${commons-configuration.version} + + + commons-daemon + commons-daemon + ${commons-daemon.version} + + + log4j + log4j + ${log4j.version} + + + org.slf4j + jcl-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + commons-httpclient + commons-httpclient + ${commons-httpclient.version} + + + com.google.guava + guava + ${guava.version} + + + com.jcraft + jsch + ${jsch.version} + + + org.dbunit + dbunit + ${dbunit.version} + + + org.apache.maven + maven-model + ${maven-model.version} + + + com.h2database + h2 + ${h2.version} + + + xerces + xercesImpl + ${xerces.version} + + + xalan + xalan + ${xalan.version} + + + com.ning + compress-lzf + ${compress-lzf.version} + + + com.n3twork.druid + extendedset + ${extendedset.version} + + + + org.quartz-scheduler + quartz + ${quartz.version} + + + org.quartz-scheduler + quartz-jobs + ${quartz.version} + + + + org.apache.curator + curator-framework + ${curator.version} + + + org.apache.curator + curator-recipes + ${curator.version} + + + + + + + + central + http://repo.maven.apache.org/maven2 + + + conjars + http://conjars.org/repo/ + + + + true + always + warn + + + false + never + fail + + HDPReleases + HDP Releases + http://repo.hortonworks.com/content/repositories/releases/ + default + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${javaVersion} + ${javaVersion} + + + + org.apache.maven.plugins + maven-site-plugin + 2.0 + + + org.apache.maven.plugins + maven-install-plugin + 2.2 + + + org.apache.maven.plugins + maven-resources-plugin + 2.4 + + UTF-8 + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-war-plugin + 2.0.2 + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-antrun-plugin + + + org.apache.maven.plugins + maven-source-plugin + 2.1.2 + + + attach-sources + package + + jar-no-fork + + + + + true + + + + org.antlr + antlr3-maven-plugin + ${antlr.version} + + + + + antlr + + false + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.6 + + + + integration-test + verify + + + + + + org.apache.maven.plugins + maven-release-plugin + 2.5.1 + + + org.jacoco + jacoco-maven-plugin + 0.7.0.201403182114 + + ${sonar.jacoco.reportPath} + true + + + + agent + + prepare-agent + + + + + + + + + + scm:git:git@github.com:KylinOLAP/Kylin.git + scm:git:git@github.com:KylinOLAP/Kylin.git + scm:git:git@github.com:KylinOLAP/Kylin.git + + + + common + metadata + dictionary + cube + job + storage + query + server + jdbc + + + + default + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.16 + + + **/BuildCubeWithEngineTest.java + **/SampleCubeSetupTest.java + **/KylinQueryTest.java + + + + + + + + + e2e + + + diff --git a/query/pom.xml b/query/pom.xml index 927830a..b13066d 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -1,186 +1,186 @@ - - 4.0.0 - - kylin-query - jar - Kylin:Query - kylin query engine based on optiq - - - com.kylinolap - kylin - 0.6.2-SNAPSHOT - - - - - - - - - com.kylinolap - kylin-storage - ${project.parent.version} - - - org.apache.calcite - calcite-core - - - org.apache.calcite - calcite-avatica - - - net.hydromatic - linq4j - - - log4j - log4j - - - org.slf4j - jcl-over-slf4j - - - org.slf4j - slf4j-api - - - commons-cli - commons-cli - - - - - - xerces - xercesImpl - test - - - xalan - xalan - test - - - com.h2database - h2 - test - - - org.dbunit - dbunit - test - - - org.apache.hadoop - hadoop-common - provided - - - org.apache.hadoop - hadoop-mapreduce-client-core - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - - com.google.protobuf - protobuf-java - - - - - org.apache.hbase - hbase-client - provided - - - org.apache.hbase - hbase-server - provided - - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - - - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - provided - - - org.apache.hive - hive-jdbc - provided - - - - - - - - - - + + 4.0.0 + + kylin-query + jar + Kylin:Query + kylin query engine based on optiq + + + com.kylinolap + kylin + 0.6.2 + + + + + + + + + com.kylinolap + kylin-storage + ${project.parent.version} + + + org.apache.calcite + calcite-core + + + org.apache.calcite + calcite-avatica + + + net.hydromatic + linq4j + + + log4j + log4j + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + commons-cli + commons-cli + + + + + + xerces + xercesImpl + test + + + xalan + xalan + test + + + com.h2database + h2 + test + + + org.dbunit + dbunit + test + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + + com.google.protobuf + protobuf-java + + + + + org.apache.hbase + hbase-client + provided + + + org.apache.hbase + hbase-server + provided + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + provided + + + org.apache.hive + hive-jdbc + provided + + + + + + + + + + diff --git a/server/pom.xml b/server/pom.xml index 523a123..8879043 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -1,390 +1,390 @@ - - 4.0.0 - - kylin-server - war - Kylin:RESTServer - Kylin REST Service and RPC Server. - - - com.kylinolap - kylin - 0.6.2-SNAPSHOT - - - - com.kylinolap.odbc.Application - - - - - com.kylinolap - kylin-query - ${project.parent.version} - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - - - com.kylinolap - kylin-job - ${project.parent.version} - - - org.apache.hbase - hbase-common - - - - - - xerces - xercesImpl - - - xalan - xalan - - - - org.springframework.boot - spring-boot-starter-web - 0.5.0.M6 - - - org.springframework.boot - spring-boot-starter-tomcat - - - ch.qos.logback - logback-classic - - - - - com.fasterxml.jackson.core - jackson-databind - - - - log4j - log4j - - - org.slf4j - jcl-over-slf4j - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - org.springframework - spring-beans - ${spring.framework.version} - - - org.springframework - spring-webmvc - ${spring.framework.version} - - - org.springframework - spring-jdbc - ${spring.framework.version} - - - org.springframework - spring-web - ${spring.framework.version} - jar - - - org.springframework - spring-core - ${spring.framework.version} - jar - - - org.springframework - spring-aop - ${spring.framework.version} - jar - - - org.springframework - spring-test - ${spring.framework.version} - - - - - org.springframework.security - spring-security-core - ${spring.framework.version} - - - org.springframework.security - spring-security-web - ${spring.framework.version} - - - org.springframework.security - spring-security-config - ${spring.framework.version} - - - org.springframework.security - spring-security-acl - ${spring.framework.version} - - - org.springframework.security - spring-security-ldap - ${spring.framework.version} - - - net.sf.ehcache - ehcache - 2.8.1 - - - com.codahale.metrics - metrics-core - ${metrics.version} - - - com.codahale.metrics - metrics-servlets - ${metrics.version} - - - com.codahale.metrics - metrics-servlet - ${metrics.version} - - - com.codahale.metrics - metrics-ganglia - ${metrics.version} - - - com.ryantenney.metrics - metrics-spring - 3.0.0 - - - cglib - cglib - 2.2.2 - - - net.sf.supercsv - super-csv - 2.1.0 - - - com.h2database - h2 - - - - - org.aspectj - aspectjrt - 1.6.11 - - - org.aspectj - aspectjweaver - 1.6.11 - - - - - org.apache.hadoop - hadoop-yarn-server-resourcemanager - provided - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - - - org.apache.hadoop - hadoop-common - provided - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - provided - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - - - org.apache.hadoop - hadoop-hdfs - provided - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - - - org.apache.hbase - hbase-client - provided - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - - - org.apache.hbase - hbase-server - provided - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - org.mortbay.jetty - servlet-api-2.5 - - - javax.servlet.jsp - jsp-api - - - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - provided - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - - - - - org.apache.tomcat - tomcat-catalina - 7.0.52 - provided - - - org.apache.tomcat - tomcat-jasper - 7.0.52 - provided - - - org.apache.tomcat.embed - tomcat-embed-core - 7.0.52 - provided - - - junit - junit - test - - - - - - - maven-compiler-plugin - - - org.springframework.boot - spring-boot-maven-plugin - - - - - - - spring-snapshots - http://repo.spring.io/libs-snapshot - - true - - - - mvn-adamgent - http://mvn-adamgent.googlecode.com/svn/maven/release - Adam Gent Maven Repository - - - - - spring-snapshots - http://repo.spring.io/libs-snapshot - - true - - - - + + 4.0.0 + + kylin-server + war + Kylin:RESTServer + Kylin REST Service and RPC Server. + + + com.kylinolap + kylin + 0.6.2 + + + + com.kylinolap.odbc.Application + + + + + com.kylinolap + kylin-query + ${project.parent.version} + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + com.kylinolap + kylin-job + ${project.parent.version} + + + org.apache.hbase + hbase-common + + + + + + xerces + xercesImpl + + + xalan + xalan + + + + org.springframework.boot + spring-boot-starter-web + 0.5.0.M6 + + + org.springframework.boot + spring-boot-starter-tomcat + + + ch.qos.logback + logback-classic + + + + + com.fasterxml.jackson.core + jackson-databind + + + + log4j + log4j + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + org.springframework + spring-beans + ${spring.framework.version} + + + org.springframework + spring-webmvc + ${spring.framework.version} + + + org.springframework + spring-jdbc + ${spring.framework.version} + + + org.springframework + spring-web + ${spring.framework.version} + jar + + + org.springframework + spring-core + ${spring.framework.version} + jar + + + org.springframework + spring-aop + ${spring.framework.version} + jar + + + org.springframework + spring-test + ${spring.framework.version} + + + + + org.springframework.security + spring-security-core + ${spring.framework.version} + + + org.springframework.security + spring-security-web + ${spring.framework.version} + + + org.springframework.security + spring-security-config + ${spring.framework.version} + + + org.springframework.security + spring-security-acl + ${spring.framework.version} + + + org.springframework.security + spring-security-ldap + ${spring.framework.version} + + + net.sf.ehcache + ehcache + 2.8.1 + + + com.codahale.metrics + metrics-core + ${metrics.version} + + + com.codahale.metrics + metrics-servlets + ${metrics.version} + + + com.codahale.metrics + metrics-servlet + ${metrics.version} + + + com.codahale.metrics + metrics-ganglia + ${metrics.version} + + + com.ryantenney.metrics + metrics-spring + 3.0.0 + + + cglib + cglib + 2.2.2 + + + net.sf.supercsv + super-csv + 2.1.0 + + + com.h2database + h2 + + + + + org.aspectj + aspectjrt + 1.6.11 + + + org.aspectj + aspectjweaver + 1.6.11 + + + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + provided + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + org.apache.hadoop + hadoop-common + provided + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + org.apache.hadoop + hadoop-hdfs + provided + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + org.apache.hbase + hbase-client + provided + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + org.apache.hbase + hbase-server + provided + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + servlet-api-2.5 + + + javax.servlet.jsp + jsp-api + + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + provided + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + + + org.apache.tomcat + tomcat-catalina + 7.0.52 + provided + + + org.apache.tomcat + tomcat-jasper + 7.0.52 + provided + + + org.apache.tomcat.embed + tomcat-embed-core + 7.0.52 + provided + + + junit + junit + test + + + + + + + maven-compiler-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + + spring-snapshots + http://repo.spring.io/libs-snapshot + + true + + + + mvn-adamgent + http://mvn-adamgent.googlecode.com/svn/maven/release + Adam Gent Maven Repository + + + + + spring-snapshots + http://repo.spring.io/libs-snapshot + + true + + + + diff --git a/storage/pom.xml b/storage/pom.xml index 146d03a..042566f 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -1,101 +1,101 @@ - - 4.0.0 - - kylin-storage - Kylin:Storage - - - com.kylinolap - kylin - 0.6.2-SNAPSHOT - - - - - com.kylinolap - kylin-cube - ${project.parent.version} - - - - - - org.apache.hbase - hbase-client - provided - - - org.apache.hbase - hbase-server - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - - com.google.protobuf - protobuf-java - - - - - - junit - junit - test - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.2 - - - package - - shade - - - true - coprocessor - - - - - - com.kylinolap:kylin-common - com.kylinolap:kylin-metadata - com.kylinolap:kylin-dictionary - com.kylinolap:kylin-cube - com.kylinolap:kylin-storage - net.sf.trove4j:* - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - + + 4.0.0 + + kylin-storage + Kylin:Storage + + + com.kylinolap + kylin + 0.6.2 + + + + + com.kylinolap + kylin-cube + ${project.parent.version} + + + + + + org.apache.hbase + hbase-client + provided + + + org.apache.hbase + hbase-server + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + + com.google.protobuf + protobuf-java + + + + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.2 + + + package + + shade + + + true + coprocessor + + + + + + com.kylinolap:kylin-common + com.kylinolap:kylin-metadata + com.kylinolap:kylin-dictionary + com.kylinolap:kylin-cube + com.kylinolap:kylin-storage + net.sf.trove4j:* + + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + \ No newline at end of file From 6ffa694b0501235ddf3403881c40ed4257e2ba65 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Thu, 20 Nov 2014 18:47:37 +0800 Subject: [PATCH 10/24] auto merge of incremental build on hll measures --- .../main/java/com/kylinolap/cube/CubeInstance.java | 33 +++++++++++ .../main/java/com/kylinolap/cube/CubeManager.java | 66 ++++++++++++---------- .../com/kylinolap/cube/CubeSegmentValidator.java | 41 +++++++------- .../java/com/kylinolap/job/JobInstanceBuilder.java | 21 +++++-- .../java/com/kylinolap/job/JoinedFlatTable.java | 4 ++ .../com/kylinolap/job/flow/JobFlowListener.java | 7 +++ .../com/kylinolap/rest/service/JobService.java | 14 ++++- 7 files changed, 131 insertions(+), 55 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java index 199e9fb..4a3796a 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; import com.kylinolap.common.KylinConfig; import com.kylinolap.common.persistence.ResourceStore; import com.kylinolap.common.persistence.RootPersistentEntity; @@ -322,6 +323,16 @@ public CubeSegment getLatestReadySegment() { return segments; } + public List getSegment(CubeSegmentStatusEnum status) { + List result = Lists.newArrayList(); + for (CubeSegment segment: getSegments()) { + if (segment.getStatus() == status) { + result.add(segment); + } + } + return result; + } + public CubeSegment getSegment(String name, CubeSegmentStatusEnum status) { for (CubeSegment segment : this.getSegments()) { if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) { @@ -344,4 +355,26 @@ public void setCreateTime(String createTime) { this.createTime = createTime; } + public long[] getDateRange() { + List readySegments = getSegment(CubeSegmentStatusEnum.READY); + if (readySegments.isEmpty()) { + return new long[]{0L, 0L}; + } + long start = Long.MAX_VALUE; + long end = Long.MIN_VALUE; + for (CubeSegment segment: readySegments) { + if (segment.getDateRangeStart() < start) { + start = segment.getDateRangeStart(); + } + if (segment.getDateRangeEnd() > end) { + end = segment.getDateRangeEnd(); + } + } + return new long[]{start, end}; + } + + public boolean incrementalBuildOnHll() { + return (!getSegment(CubeSegmentStatusEnum.READY).isEmpty()) && true;//FIXME && true for test only + } + } diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java index 3578c27..3fb7362 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java @@ -17,12 +17,8 @@ package com.kylinolap.cube; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.text.SimpleDateFormat; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.lang3.StringUtils; @@ -266,33 +262,39 @@ public CubeInstance updateCube(CubeInstance cube) throws IOException { public List allocateSegments(CubeInstance cubeInstance, CubeBuildTypeEnum buildType, long startDate, long endDate) throws IOException, CubeIntegrityException { if (cubeInstance.getBuildingSegments().size() > 0) { - throw new RuntimeException("There is already a allocating segment!"); + throw new RuntimeException("There is already an allocating segment!"); } List segments = new ArrayList(); if (null != cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateColumn()) { - if (startDate == 0 && cubeInstance.getSegments().size() == 0) { - startDate = cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(); - } + if (cubeInstance.incrementalBuildOnHll()) { + long[] dateRange = cubeInstance.getDateRange(); + segments.add(buildSegment(cubeInstance, dateRange[0], endDate)); + } else { - // incremental build - CubeSegment lastSegment = null; - for (CubeSegment segment : cubeInstance.getSegments()) { - if (segment.getDateRangeStart() == startDate) { - // refresh or merge - segments.add(buildSegment(cubeInstance, startDate, endDate)); + if (startDate == 0 && cubeInstance.getSegments().size() == 0) { + startDate = cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart(); } - if (segment.getDateRangeStart() < startDate && startDate < segment.getDateRangeEnd()) { - // delete-insert - segments.add(buildSegment(cubeInstance, segment.getDateRangeStart(), startDate)); - segments.add(buildSegment(cubeInstance, startDate, endDate)); + + // incremental build + CubeSegment lastSegment = null; + for (CubeSegment segment : cubeInstance.getSegments()) { + if (segment.getDateRangeStart() == startDate) { + // refresh or merge + segments.add(buildSegment(cubeInstance, startDate, endDate)); + } + if (segment.getDateRangeStart() < startDate && startDate < segment.getDateRangeEnd()) { + // delete-insert + segments.add(buildSegment(cubeInstance, segment.getDateRangeStart(), startDate)); + segments.add(buildSegment(cubeInstance, startDate, endDate)); + } + lastSegment = segment; } - lastSegment = segment; - } - // append - if (null == lastSegment || (lastSegment.getDateRangeEnd() == startDate)) { - segments.add(buildSegment(cubeInstance, startDate, endDate)); + // append + if (null == lastSegment || (lastSegment.getDateRangeEnd() == startDate)) { + segments.add(buildSegment(cubeInstance, startDate, endDate)); + } } } else { segments.add(buildSegment(cubeInstance, 0, 0)); @@ -300,7 +302,7 @@ public CubeInstance updateCube(CubeInstance cube) throws IOException { validateNewSegments(cubeInstance, buildType, segments); - if (buildType == CubeBuildTypeEnum.MERGE) { + if (buildType == CubeBuildTypeEnum.MERGE || cubeInstance.incrementalBuildOnHll()) { this.makeDictForNewSegment(cubeInstance, segments.get(0)); this.makeSnapshotForNewSegment(cubeInstance, segments.get(0)); } @@ -333,10 +335,14 @@ public void updateSegmentOnJobSucceed(CubeInstance cubeInstance, CubeBuildTypeEn switch (buildType) { case BUILD: - if (segmentsInNewStatus.size() == 1) {// if this the last segment in - // status of NEW - // remove all the rebuilding/impacted segments - cubeInstance.getSegments().removeAll(cubeInstance.getRebuildingSegments()); + if (cubeInstance.incrementalBuildOnHll()) { + cubeInstance.getSegments().removeAll(cubeInstance.getMergingSegments()); + } else { + if (segmentsInNewStatus.size() == 1) {// if this the last segment in + // status of NEW + // remove all the rebuilding/impacted segments + cubeInstance.getSegments().removeAll(cubeInstance.getRebuildingSegments()); + } } break; case MERGE: diff --git a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java index d2e69c9..5a3c6d8 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeSegmentValidator.java @@ -154,29 +154,32 @@ void validate(CubeInstance cubeInstance, List newSegments) throws C if (newSegments.size() != 1) { throw new CubeIntegrityException("Invalid date range."); } + if (cubeInstance.incrementalBuildOnHll()) { - CubeSegment newSegment = newSegments.get(0); - // check if user will rebuild one specified segment - boolean hasMatchSegment = false; - for (CubeSegment segment : cubeInstance.getSegments()) { - if (segment.getDateRangeStart() == newSegment.getDateRangeStart()) { - if (segment.getDateRangeEnd() == newSegment.getDateRangeEnd()) { - hasMatchSegment = true; - } else { - throw new CubeIntegrityException("Invalid date range."); + } else { + CubeSegment newSegment = newSegments.get(0); + // check if user will rebuild one specified segment + boolean hasMatchSegment = false; + for (CubeSegment segment : cubeInstance.getSegments()) { + if (segment.getDateRangeStart() == newSegment.getDateRangeStart()) { + if (segment.getDateRangeEnd() == newSegment.getDateRangeEnd()) { + hasMatchSegment = true; + } else { + throw new CubeIntegrityException("Invalid date range."); + } } } - } - if (!hasMatchSegment) { - if (cubeInstance.getSegments().size() == 0) { - if (cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart() != newSegment.getDateRangeStart()) { - throw new CubeIntegrityException("Invalid start date."); - } - } else { - CubeSegment lastSegment = cubeInstance.getSegments().get(cubeInstance.getSegments().size() - 1); - if (newSegment.getDateRangeStart() != lastSegment.getDateRangeEnd()) { - throw new CubeIntegrityException("Invalid start date."); + if (!hasMatchSegment) { + if (cubeInstance.getSegments().size() == 0) { + if (cubeInstance.getDescriptor().getCubePartitionDesc().getPartitionDateStart() != newSegment.getDateRangeStart()) { + throw new CubeIntegrityException("Invalid start date."); + } + } else { + CubeSegment lastSegment = cubeInstance.getSegments().get(cubeInstance.getSegments().size() - 1); + if (newSegment.getDateRangeStart() != lastSegment.getDateRangeEnd()) { + throw new CubeIntegrityException("Invalid start date."); + } } } } diff --git a/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java b/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java index 1cd9195..7f5a6a8 100644 --- a/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java +++ b/job/src/main/java/com/kylinolap/job/JobInstanceBuilder.java @@ -162,7 +162,7 @@ private String getRowkeyDistributionOutputPath() { // jobInstance.clearSteps(); int stepSeqNum = 0; List result = Lists.newArrayList(); - final String mergedCuboidPath = jobWorkingDir + "/" + cubeName + "/merged_cuboid"; + final String mergedCuboidPath = jobWorkingDir + "/" + cubeName + "/cuboid"; // merge cuboid data of ancestor segments result.add(createMergeCuboidDataStep(jobInstance, stepSeqNum++, formattedPath, mergedCuboidPath)); @@ -212,8 +212,10 @@ private String getRowkeyDistributionOutputPath() { result.add(createBuildDictionaryStep(jobInstance, stepSeqNum++)); final String cuboidRootPath = jobWorkingDir + "/" + cubeName + "/cuboid/"; + final String cuboidTmpRootPath = jobWorkingDir + "/" + cubeName + "/tmp_cuboid/"; + final boolean incBuildMerge = cube.incrementalBuildOnHll(); - String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount); + String[] cuboidOutputTempPath = getCuboidOutputPaths(incBuildMerge?cuboidTmpRootPath:cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount); // base cuboid step result.add(createBaseCuboidStep(jobInstance, stepSeqNum++, cuboidOutputTempPath)); @@ -223,13 +225,24 @@ private String getRowkeyDistributionOutputPath() { result.add(createNDimensionCuboidStep(jobInstance, stepSeqNum++, cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount)); } + if (incBuildMerge) { + List pathToMerge = Lists.newArrayList(); + for (CubeSegment segment: cube.getSegments(CubeSegmentStatusEnum.READY)) { + String path = JobInstance.getJobWorkingDir(segment.getLastBuildJobID(), engineConfig.getHdfsWorkingDirectory()) + "/" + jobInstance.getRelatedCube() + "/cuboid/*"; + pathToMerge.add(path); + } + pathToMerge.add(cuboidTmpRootPath + "*"); + result.add(createMergeCuboidDataStep(jobInstance, stepSeqNum++, formatPaths(pathToMerge), cuboidRootPath)); + } + String cuboidPath = incBuildMerge?cuboidRootPath:cuboidRootPath+"*"; + // get output distribution step - result.add(createRangeRowkeyDistributionStep(jobInstance, stepSeqNum++, cuboidRootPath + "*")); + result.add(createRangeRowkeyDistributionStep(jobInstance, stepSeqNum++, cuboidPath)); // create htable step result.add(createCreateHTableStep(jobInstance, stepSeqNum++)); // generate hfiles step - result.add(createConvertCuboidToHfileStep(jobInstance, stepSeqNum++, cuboidRootPath + "*")); + result.add(createConvertCuboidToHfileStep(jobInstance, stepSeqNum++, cuboidPath)); // bulk load step result.add(createBulkLoadStep(jobInstance, stepSeqNum++)); diff --git a/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java b/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java index 2a411c1..053dc5b 100644 --- a/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java +++ b/job/src/main/java/com/kylinolap/job/JoinedFlatTable.java @@ -186,6 +186,10 @@ private static void appendWhereStatement(JoinedFlatTableDesc intermediateTableDe if (null != cubeSegment) { long dateStart = cubeSegment.getDateRangeStart(); long dateEnd = cubeSegment.getDateRangeEnd(); + + if (cubeSegment.getCubeInstance().incrementalBuildOnHll()) { + dateStart = cubeSegment.getCubeInstance().getDateRange()[1]; + } if (!(dateStart == 0 && dateEnd == 0)) { String partitionColumnName = cubeDesc.getCubePartitionDesc().getPartitionDateColumn(); diff --git a/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java b/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java index f101f4d..eea0d0b 100644 --- a/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java +++ b/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java @@ -277,6 +277,13 @@ private void updateCubeSegmentInfoOnSucceed(JobInstance jobInstance, JobEngineCo } else { log.info("No step with name '" + JobConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE + "' is found"); } + + if (cubeInstance.incrementalBuildOnHll()) { + for (CubeSegment seg : cubeInstance.getMergingSegments()) { + sourceCount += seg.getSourceRecords(); + sourceSize += seg.getSourceRecordsSize(); + } + } break; case MERGE: for (CubeSegment seg : cubeInstance.getMergingSegments()) { diff --git a/server/src/main/java/com/kylinolap/rest/service/JobService.java b/server/src/main/java/com/kylinolap/rest/service/JobService.java index d5c9ed8..042f44f 100644 --- a/server/src/main/java/com/kylinolap/rest/service/JobService.java +++ b/server/src/main/java/com/kylinolap/rest/service/JobService.java @@ -22,6 +22,8 @@ import java.util.Date; import java.util.List; +import com.google.common.collect.Lists; +import com.kylinolap.cube.CubeManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -107,9 +109,17 @@ public String submitJob(CubeInstance cube, long startDate, long endDate, CubeBui String uuid = null; try { - for (CubeSegment segment : this.getCubeManager().allocateSegments(cube, buildType, startDate, endDate)) { + List cubeSegments = this.getCubeManager().allocateSegments(cube, buildType, startDate, endDate); + List jobs = Lists.newArrayListWithExpectedSize(cubeSegments.size()); + for (CubeSegment segment : cubeSegments) { JobInstance job = this.getJobManager().createJob(cube.getName(), segment.getName(), buildType); - uuid = this.getJobManager().submitJob(job); + jobs.add(job); + uuid = job.getUuid(); + segment.setLastBuildJobID(uuid); + } + getCubeManager().updateCube(cube); + for (JobInstance job: jobs) { + this.getJobManager().submitJob(job); permissionService.init(job, null); permissionService.inherit(job, cube); } From 3f6949c15f9559e8aae6e4a0614b9ccb58af0e7b Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 20 Nov 2014 18:59:07 +0800 Subject: [PATCH 11/24] [maven-release-plugin] rollback the release of kylin-0.6.2 --- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 18 ++++++++---------- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 10 files changed, 17 insertions(+), 19 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index baaf4c8..7a7555f 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.2-SNAPSHOT diff --git a/cube/pom.xml b/cube/pom.xml index 173febd..bf5fb0f 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.2-SNAPSHOT diff --git a/dictionary/pom.xml b/dictionary/pom.xml index 2cdf37f..8d57c90 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.2-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 593553d..9bfe79e 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.2-SNAPSHOT diff --git a/job/pom.xml b/job/pom.xml index 525fc8d..e596a7b 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.2-SNAPSHOT kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index 59015c3..abafce5 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.2-SNAPSHOT diff --git a/pom.xml b/pom.xml index 31802e9..3ae260e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,13 +4,12 @@ com.kylinolap kylin pom - 0.6.2 + 0.6.2-SNAPSHOT Kylin:HadoopOLAPEngine 1.7 - 3.1.1 UTF-8 UTF-8 @@ -23,28 +22,27 @@ 3.4 - - 4.11 - 2.5.0 1.0.0 - 1.3.174 - 1.2 2.6 3.1 2.4 1.9 1.0.15 - 3.1 - + 1.2.17 1.6.4 2.2.3 + 3.1 12.0.1 + 0.1.51 + 2.5.0 + 3.1.1 + 1.3.174 2.9.1 2.7.1 0.8.4 @@ -467,7 +465,7 @@ org.apache.maven.plugins maven-release-plugin - 2.5.1 + 2.1 org.jacoco diff --git a/query/pom.xml b/query/pom.xml index b13066d..9cb1746 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.2-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 8879043..e0504fe 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.2-SNAPSHOT diff --git a/storage/pom.xml b/storage/pom.xml index 042566f..c6908b5 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.2-SNAPSHOT From 8931f674e3e8fecd456bf4e3365e7df0798b2c59 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 20 Nov 2014 19:01:01 +0800 Subject: [PATCH 12/24] prepare release --- docs/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/README.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..e69de29 From 7b8ccad73a71ec778b63325bee3db653cef06e10 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 20 Nov 2014 20:15:54 +0800 Subject: [PATCH 13/24] prepare release --- pom.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 3ae260e..2f6241e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,6 +10,7 @@ 1.7 + 3.1.1 UTF-8 UTF-8 @@ -24,25 +25,24 @@ 3.4 4.11 1.0.0 + 2.5.0 + 1.3.174 + 1.2 2.6 3.1 2.4 1.9 1.0.15 + 3.1 - + 1.2.17 1.6.4 2.2.3 - 3.1 12.0.1 - 0.1.51 - 2.5.0 - 3.1.1 - 1.3.174 2.9.1 2.7.1 0.8.4 From 4a986a2e653b16915eb8a3d50737d35f31319c09 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 20 Nov 2014 20:29:57 +0800 Subject: [PATCH 14/24] fix maven release API incompatibility --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2f6241e..80d4a9f 100644 --- a/pom.xml +++ b/pom.xml @@ -11,6 +11,7 @@ 1.7 3.1.1 + 2.5.1 UTF-8 UTF-8 @@ -465,7 +466,7 @@ org.apache.maven.plugins maven-release-plugin - 2.1 + ${maven-release.version} org.jacoco From ca6ad3251131273ec0dc707fc0a9b590a643fa2d Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 20 Nov 2014 20:31:11 +0800 Subject: [PATCH 15/24] [maven-release-plugin] prepare release kylin-0.6.2 --- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 5 +++-- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 10 files changed, 12 insertions(+), 11 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 7a7555f..baaf4c8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.2-SNAPSHOT + 0.6.2 diff --git a/cube/pom.xml b/cube/pom.xml index bf5fb0f..173febd 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.2-SNAPSHOT + 0.6.2 diff --git a/dictionary/pom.xml b/dictionary/pom.xml index 8d57c90..2cdf37f 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.2-SNAPSHOT + 0.6.2 diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 9bfe79e..593553d 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.2-SNAPSHOT + 0.6.2 diff --git a/job/pom.xml b/job/pom.xml index e596a7b..525fc8d 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.2-SNAPSHOT + 0.6.2 kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index abafce5..59015c3 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.2-SNAPSHOT + 0.6.2 diff --git a/pom.xml b/pom.xml index 80d4a9f..de0a847 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin pom - 0.6.2-SNAPSHOT + 0.6.2 Kylin:HadoopOLAPEngine @@ -493,7 +493,8 @@ scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git - + kylin-0.6.2 + common diff --git a/query/pom.xml b/query/pom.xml index 9cb1746..b13066d 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.2-SNAPSHOT + 0.6.2 diff --git a/server/pom.xml b/server/pom.xml index e0504fe..8879043 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.2-SNAPSHOT + 0.6.2 diff --git a/storage/pom.xml b/storage/pom.xml index c6908b5..042566f 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.2-SNAPSHOT + 0.6.2 From ad8d66c58e78ba5d691f0e5e0e8a53a02edb7b23 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 20 Nov 2014 20:31:32 +0800 Subject: [PATCH 16/24] [maven-release-plugin] prepare for next development iteration --- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 4 ++-- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index baaf4c8..654df7e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.3-SNAPSHOT diff --git a/cube/pom.xml b/cube/pom.xml index 173febd..f177c07 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.3-SNAPSHOT diff --git a/dictionary/pom.xml b/dictionary/pom.xml index 2cdf37f..67310cb 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.3-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 593553d..e69cac0 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.3-SNAPSHOT diff --git a/job/pom.xml b/job/pom.xml index 525fc8d..b039957 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.3-SNAPSHOT kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index 59015c3..3e76619 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.3-SNAPSHOT diff --git a/pom.xml b/pom.xml index de0a847..9e946b7 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin pom - 0.6.2 + 0.6.3-SNAPSHOT Kylin:HadoopOLAPEngine @@ -493,7 +493,7 @@ scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git - kylin-0.6.2 + HEAD diff --git a/query/pom.xml b/query/pom.xml index b13066d..189eb82 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.3-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 8879043..fea3397 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.3-SNAPSHOT diff --git a/storage/pom.xml b/storage/pom.xml index 042566f..ce8d1c7 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.2 + 0.6.3-SNAPSHOT From 6f55338b9b343819d67df1a71c7e6cb0708a01a7 Mon Sep 17 00:00:00 2001 From: Olivier Guennec Date: Thu, 20 Nov 2014 07:38:46 -0800 Subject: [PATCH 17/24] Enabling cancellation of PENDING jobs in controller JobManager.java --- job/src/main/java/com/kylinolap/job/JobManager.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/job/src/main/java/com/kylinolap/job/JobManager.java b/job/src/main/java/com/kylinolap/job/JobManager.java index ce1bc58..2f1107f 100644 --- a/job/src/main/java/com/kylinolap/job/JobManager.java +++ b/job/src/main/java/com/kylinolap/job/JobManager.java @@ -149,6 +149,13 @@ public void discardJob(String uuid) throws IOException, CubeIntegrityException, CubeManager.getInstance(config).updateSegmentOnJobDiscard(cube, jobInstance.getRelatedSegment()); } break; + case PENDING: + try { + killRunningJob(jobInstance); + } finally { + CubeManager.getInstance(config).updateSegmentOnJobDiscard(cube, jobInstance.getRelatedSegment()); + } + break; case ERROR: try { for (JobStep jobStep : jobInstance.getSteps()) { From 9c4161271e17198f7000ea126d83940d04bfa82c Mon Sep 17 00:00:00 2001 From: Luke Han Date: Fri, 21 Nov 2014 08:09:46 +0800 Subject: [PATCH 18/24] Only admin role could sync up Hive table metadata --- webapp/app/partials/tables/source_table_tree.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/app/partials/tables/source_table_tree.html b/webapp/app/partials/tables/source_table_tree.html index d561bcd..082163b 100644 --- a/webapp/app/partials/tables/source_table_tree.html +++ b/webapp/app/partials/tables/source_table_tree.html @@ -10,7 +10,7 @@

Tables

Loading...
- +
From cbe844956a065a27d6472bfdb7dbb14de2e52140 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Fri, 21 Nov 2014 08:52:38 +0800 Subject: [PATCH 19/24] https://github.com/KylinOLAP/Kylin/issues/83 --- .../main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java index de2fe63..7820332 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java +++ b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java @@ -224,7 +224,8 @@ private static String convertType(String colType) { return "smallint"; } else if ("byte".equals(colType)) { return "tinyint"; - } + } else if ("bool".equals(colType)) + return "boolean"; return colType; } From 8e0b5dff874d786b58ec4d44a6ce90e49e425fcd Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Fri, 21 Nov 2014 15:16:10 +0800 Subject: [PATCH 20/24] fix incremental build twice on cube with holistic measures --- cube/src/main/java/com/kylinolap/cube/CubeInstance.java | 2 +- .../java/com/kylinolap/job/hadoop/AbstractHadoopJob.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java index 4a3796a..b2d7886 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java @@ -374,7 +374,7 @@ public void setCreateTime(String createTime) { } public boolean incrementalBuildOnHll() { - return (!getSegment(CubeSegmentStatusEnum.READY).isEmpty()) && true;//FIXME && true for test only + return (!getSegment(CubeSegmentStatusEnum.READY).isEmpty()) && getDescriptor().hasHolisticCountDistinctMeasures(); } } diff --git a/job/src/main/java/com/kylinolap/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/com/kylinolap/job/hadoop/AbstractHadoopJob.java index ed78fb7..4cd59a9 100644 --- a/job/src/main/java/com/kylinolap/job/hadoop/AbstractHadoopJob.java +++ b/job/src/main/java/com/kylinolap/job/hadoop/AbstractHadoopJob.java @@ -138,9 +138,17 @@ public void addInputDirs(String input, Job job) throws IOException { if (inp.endsWith("/*")) { inp = inp.substring(0, inp.length() - 2); FileSystem fs = FileSystem.get(job.getConfiguration()); - for (FileStatus stat : fs.listStatus(new Path(inp))) { - if (stat.isDirectory()) + Path path = new Path(inp); + FileStatus[] fileStatuses = fs.listStatus(path); + boolean hasDir = false; + for (FileStatus stat : fileStatuses) { + if (stat.isDirectory()) { + hasDir = true; addInputDirs(stat.getPath().toString(), job); + } + } + if (fileStatuses.length > 0 && !hasDir) { + addInputDirs(path.toString(), job); } } else { System.out.println("Add input " + inp); From d88dd99c69e3ccdaf8deedc8e52c0e7c850eada2 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Fri, 21 Nov 2014 22:09:17 +0800 Subject: [PATCH 21/24] fix typo --- webapp/app/js/controllers/cubeEdit.js | 196 ++++++++++++++++++++- .../partials/cubeDesigner/advanced_settings.html | 5 +- 2 files changed, 189 insertions(+), 12 deletions(-) diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js index 13af908..4e10a55 100644 --- a/webapp/app/js/controllers/cubeEdit.js +++ b/webapp/app/js/controllers/cubeEdit.js @@ -152,6 +152,8 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio // generate default rowkey and aggregation groups. if ($scope.cubeMetaFrame.rowkey.rowkey_columns.length == 0 && $scope.cubeMetaFrame.rowkey.aggregation_groups.length == 0) { generateDefaultRowkey(); + }else{ + reGenerateRowKey(); } // Clean up objects used in cube creation @@ -165,6 +167,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio } }); + if ($scope.cubeMetaFrame.cube_partition_desc.partition_date_start) { var dateStart = new Date($scope.cubeMetaFrame.cube_partition_desc.partition_date_start); dateStart = (dateStart.getFullYear() + "-" + (dateStart.getMonth() + 1) + "-" + dateStart.getDate()); @@ -173,7 +176,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio } $scope.state.project = $scope.cubeMetaFrame.project; - delete $scope.cubeMetaFrame.project; +// delete $scope.cubeMetaFrame.project; $scope.state.cubeSchema = angular.toJson($scope.cubeMetaFrame, true); } @@ -238,6 +241,178 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio } } + + function reGenerateRowKey(){ + var tmpRowKeyColumns = []; + var tmpAggregationItems = []; + + angular.forEach($scope.cubeMetaFrame.dimensions, function (dimension, index) { + if (dimension.column == '{FK}' && dimension.join && dimension.join.foreign_key.length > 0) { + angular.forEach(dimension.join.foreign_key, function (fk, index) { + for (var i = 0; i < tmpRowKeyColumns.length; i++) { + if(tmpRowKeyColumns[i].column == fk) + break; + } + if(i == tmpRowKeyColumns.length) { + tmpRowKeyColumns.push({ + "column": fk, + "length": 0, + "dictionary": true, + "mandatory": false + }); + } + tmpAggregationItems.push(fk); + }); + } + else if (dimension.column) { + for (var i = 0; i < tmpRowKeyColumns.length; i++) { + if(tmpRowKeyColumns.column == dimension.column) + break; + } + if(i == tmpRowKeyColumns.length) { + tmpRowKeyColumns.push({ + "column": dimension.column, + "length": 0, + "dictionary": true, + "mandatory": false + }); + } + tmpAggregationItems.push(dimension.column); + } + if (dimension.hierarchy && dimension.hierarchy.length > 0) { + angular.forEach(dimension.hierarchy, function (hierarchy, index) { + for (var i = 0; i < tmpRowKeyColumns.length; i++) { + if(tmpRowKeyColumns.column == hierarchy.column) + break; + } + if(i == tmpRowKeyColumns.length) { + tmpRowKeyColumns.push({ + "column": hierarchy.column, + "length": 0, + "dictionary": true, + "mandatory": false + }); + } + + tmpAggregationItems.push(hierarchy.column); + }); + } + + }); + + var rowkeyColumns = $scope.cubeMetaFrame.rowkey.rowkey_columns; + var newRowKeyColumns = sortSharedData(rowkeyColumns,tmpRowKeyColumns); + var increasedColumns = increasedColumn(rowkeyColumns,tmpRowKeyColumns); + newRowKeyColumns = newRowKeyColumns.concat(increasedColumns); + + //! here get the latest rowkey_columns + $scope.cubeMetaFrame.rowkey.rowkey_columns = newRowKeyColumns; + + + var aggregationGroups = $scope.cubeMetaFrame.rowkey.aggregation_groups; + // rm unused item from group + angular.forEach(aggregationGroups, function (group, index) { + for(var j = 0;j
-

Aggreation Groups

+

Aggregation Groups

@@ -110,8 +110,7 @@

Rowkeys

From ae382d9803b584562c9757f525de2470a00d81be Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Tue, 25 Nov 2014 11:22:27 +0800 Subject: [PATCH 22/24] support timestamp --- .../main/java/com/kylinolap/dict/DateStrDictionary.java | 14 ++++++++++---- .../src/main/java/com/kylinolap/storage/tuple/Tuple.java | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java b/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java index 064f177..1114546 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java +++ b/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java @@ -42,7 +42,8 @@ public class DateStrDictionary extends Dictionary { static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd"; - static final String DEFAULT_DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS"; + static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss"; + static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS"; static final private Map> threadLocalMap = new ConcurrentHashMap>(); @@ -87,10 +88,15 @@ public static Date stringToDate(String str, String pattern) { } public static long stringToMillis(String str) { - if (str.length() == 10) + if (str.length() == 10) { return stringToDate(str, DEFAULT_DATE_PATTERN).getTime(); - else - return stringToDate(str, DEFAULT_DATETIME_PATTERN).getTime(); + } else if (str.length() == 19) { + return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime(); + } else if (str.length() == 23) { + return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime(); + } else { + throw new IllegalArgumentException("there is no valid date pattern for:" + str); + } } // ============================================================================ diff --git a/storage/src/main/java/com/kylinolap/storage/tuple/Tuple.java b/storage/src/main/java/com/kylinolap/storage/tuple/Tuple.java index de2303f..8dda7c0 100644 --- a/storage/src/main/java/com/kylinolap/storage/tuple/Tuple.java +++ b/storage/src/main/java/com/kylinolap/storage/tuple/Tuple.java @@ -135,6 +135,8 @@ public static Object convertOptiqCellValue(String strValue, String dataType) { return Double.valueOf(strValue); } else if ("decimal".equals(dataType)) { return new BigDecimal(strValue); + } else if ("timestamp".equals(dataType)) { + return Long.valueOf(DateStrDictionary.stringToMillis(strValue)); } else { return strValue; } From 7fd7eb35efd9d0a6230fac52740b178e255bf506 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Tue, 25 Nov 2014 15:02:06 +0800 Subject: [PATCH 23/24] remove rule PushFilterPastProjectRule --- .../main/java/com/kylinolap/query/relnode/OLAPTableScan.java | 10 +++++----- query/src/test/resources/query/sql/query76.sql | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 query/src/test/resources/query/sql/query76.sql diff --git a/query/src/main/java/com/kylinolap/query/relnode/OLAPTableScan.java b/query/src/main/java/com/kylinolap/query/relnode/OLAPTableScan.java index 62c2fcd..9c50c36 100644 --- a/query/src/main/java/com/kylinolap/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/com/kylinolap/query/relnode/OLAPTableScan.java @@ -30,10 +30,7 @@ import org.eigenbase.rel.RelNode; import org.eigenbase.rel.RelWriter; import org.eigenbase.rel.TableAccessRelBase; -import org.eigenbase.rel.rules.PushFilterPastJoinRule; -import org.eigenbase.rel.rules.PushJoinThroughJoinRule; -import org.eigenbase.rel.rules.RemoveDistinctAggregateRule; -import org.eigenbase.rel.rules.SwapJoinRule; +import org.eigenbase.rel.rules.*; import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelOptCost; import org.eigenbase.relopt.RelOptPlanner; @@ -121,7 +118,10 @@ public void register(RelOptPlanner planner) { planner.removeRule(SwapJoinRule.INSTANCE); planner.removeRule(PushJoinThroughJoinRule.LEFT); planner.removeRule(PushJoinThroughJoinRule.RIGHT); - + + // for columns in having clause will enable table scan filter rule + // cause kylin does not depend on MPP + planner.removeRule(PushFilterPastProjectRule.INSTANCE); // distinct count will be split into a separated query that is joined with the left query planner.removeRule(RemoveDistinctAggregateRule.INSTANCE); } diff --git a/query/src/test/resources/query/sql/query76.sql b/query/src/test/resources/query/sql/query76.sql new file mode 100644 index 0000000..2bfd2a2 --- /dev/null +++ b/query/src/test/resources/query/sql/query76.sql @@ -0,0 +1,4 @@ +select lstg_format_name, sum(price) as GMV + from test_kylin_fact + group by lstg_format_name + having SLR_SEGMENT_CD > 0 From d6ce63aa315ced87031afc01e021a842b1ea4cab Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Tue, 25 Nov 2014 15:09:05 +0800 Subject: [PATCH 24/24] fix test sql --- query/src/test/resources/query/sql/query76.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query/src/test/resources/query/sql/query76.sql b/query/src/test/resources/query/sql/query76.sql index 2bfd2a2..32b1582 100644 --- a/query/src/test/resources/query/sql/query76.sql +++ b/query/src/test/resources/query/sql/query76.sql @@ -1,4 +1,4 @@ select lstg_format_name, sum(price) as GMV from test_kylin_fact - group by lstg_format_name + group by lstg_format_name, SLR_SEGMENT_CD having SLR_SEGMENT_CD > 0