From 73b019cab3934d246e0a7cea61074cf126c5858f Mon Sep 17 00:00:00 2001 From: yangzhong Date: Tue, 8 Mar 2016 09:19:00 +0800 Subject: [PATCH] KYLIN-672: Add Env and Project Info in job email notification --- .../org/apache/kylin/common/KylinConfigBase.java | 11 +++-- .../kylin/job/constant/ExecutableConstants.java | 2 +- .../java/org/apache/kylin/engine/mr/CubingJob.java | 56 +++++++++++++++++----- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 9bbcbb4..6f1fccc 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -173,10 +173,7 @@ public class KylinConfigBase implements Serializable { if (!root.endsWith("/")) { root += "/"; } - return new StringBuffer(root) - .append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')) - .append("/") - .toString(); + return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString(); } public CliCommandExecutor getCliCommandExecutor() throws IOException { @@ -485,7 +482,7 @@ public class KylinConfigBase implements Serializable { public boolean isGetJobStatusWithKerberos() { return Boolean.valueOf(this.getOptional("kylin.job.status.with.kerberos", "false")); } - + public boolean isTransformPathToMasterNN() { return Boolean.valueOf(this.getOptional("kylin.transform.hdfs.path.enable", "false")); } @@ -526,6 +523,10 @@ public class KylinConfigBase implements Serializable { return getOptional("mail.sender", ""); } + public String getDeployEnv() { + return getOptional("deploy.env", "DEV"); + } + public String toString() { return getMetadataUrl(); } diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java index d370b0d..ab2fed7 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java @@ -75,5 +75,5 @@ public final class ExecutableConstants { public static final String PROP_JOB_KILLED = "jobKilled"; public static final String PROP_JOB_RUNTIME_FLOWS = "jobFlows"; - public static final String NOTIFY_EMAIL_TEMPLATE = "
Build Result of Job ${job_name}
    " + "
  • Build Result: ${result}
    • " + "
  • Job Engine: ${job_engine}
    • " + "
  • Cube Name: ${cube_name}
    • " + "
  • Source Records Count: ${source_records_count}
    • " + "
  • Start Time: ${start_time}
    • " + "
  • Duration: ${duration}
    • " + "
  • MR Waiting: ${mr_waiting}
    • " + "
  • Last Update Time: ${last_update_time}
    • " + "
  • Submitter: ${submitter}
    • " + "
  • Error Log: ${error_log}
    • " + "
"; + public static final String NOTIFY_EMAIL_TEMPLATE = "
Build Result of Job ${job_name}
    " + "
  • Build Result: ${result}
    • " + "
  • Job Engine: ${job_engine}
    • " + "
  • Env: ${env_name}
    • " + "
  • Project: ${project_name}
    • " + "
  • Cube Name: ${cube_name}
    • " + "
  • Source Records Count: ${source_records_count}
    • " + "
  • Start Time: ${start_time}
    • " + "
  • Duration: ${duration}
    • " + "
  • MR Waiting: ${mr_waiting}
    • " + "
  • Last Update Time: ${last_update_time}
    • " + "
  • Submitter: ${submitter}
    • " + "
  • Error Log: ${error_log}
    • " + "
"; } diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java index 979ff75..1353fe8 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java @@ -30,6 +30,7 @@ import java.util.regex.Matcher; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; @@ -43,11 +44,13 @@ import org.apache.kylin.job.execution.ExecutableContext; import org.apache.kylin.job.execution.ExecutableState; import org.apache.kylin.job.execution.ExecuteResult; import org.apache.kylin.job.execution.Output; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; /** */ public class CubingJob extends DefaultChainedExecutable { - + public static enum AlgorithmEnum { LAYER, INMEM } @@ -58,6 +61,8 @@ public class CubingJob extends DefaultChainedExecutable { public static final String CUBE_SIZE_BYTES = "byteSizeBytes"; public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime"; + private static final String DEPLOY_ENV_NAME = "envName"; + private static final String PROJECT_INSTANCE_NAME = "projectName"; private static final String CUBE_INSTANCE_NAME = "cubeName"; private static final String SEGMENT_ID = "segmentId"; @@ -70,10 +75,21 @@ public class CubingJob extends DefaultChainedExecutable { } private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) { + KylinConfig kylinConfig = config.getConfig(); + CubeInstance cube = seg.getCubeInstance(); + List projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),cube.getName()); + if(projList==null || projList.size()==0){ + throw new RuntimeException("Cannot find the project containing the cube " +cube.getName()+"!!!"); + }else if(projList.size()>=2){ + throw new RuntimeException("Find more than one project containing the cube "+cube.getName()+". It does't meet the uniqueness requirement!!! "); + } + CubingJob result = new CubingJob(); SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone())); - result.setCubeName(seg.getCubeInstance().getName()); + result.setDeployEnvName(kylinConfig.getDeployEnv()); + result.setProjectName(projList.get(0).getName()); + result.setCubeName(cube.getName()); result.setSegmentId(seg.getUuid()); result.setName(seg.getCubeInstance().getName() + " - " + seg.getName() + " - " + jobType + " - " + format.format(new Date(System.currentTimeMillis()))); result.setSubmitter(submitter); @@ -85,6 +101,22 @@ public class CubingJob extends DefaultChainedExecutable { super(); } + void setDeployEnvName(String name) { + setParam(DEPLOY_ENV_NAME, name); + } + + public String getDeployEnvName() { + return getParam(DEPLOY_ENV_NAME); + } + + void setProjectName(String name) { + setParam(PROJECT_INSTANCE_NAME, name); + } + + public String getProjectName() { + return getParam(PROJECT_INSTANCE_NAME); + } + void setCubeName(String name) { setParam(CUBE_INSTANCE_NAME, name); } @@ -131,6 +163,8 @@ public class CubingJob extends DefaultChainedExecutable { String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE; content = content.replaceAll("\\$\\{job_name\\}", getName()); content = content.replaceAll("\\$\\{result\\}", state.toString()); + content = content.replaceAll("\\$\\{env_name\\}", getDeployEnvName()); + content = content.replaceAll("\\$\\{project_name\\}", getProjectName()); content = content.replaceAll("\\$\\{cube_name\\}", getCubeName()); content = content.replaceAll("\\$\\{source_records_count\\}", String.valueOf(findSourceRecordCount())); content = content.replaceAll("\\$\\{start_time\\}", new Date(getStartTime()).toString()); @@ -147,7 +181,7 @@ public class CubingJob extends DefaultChainedExecutable { logger.warn(e.getLocalizedMessage(), e); } - String title = "[" + state.toString() + "] - [Kylin Cube Build Job]-" + getCubeName(); + String title = "[" + state.toString() + "] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - " + getCubeName(); return Pair.of(title, content); } @@ -174,11 +208,11 @@ public class CubingJob extends DefaultChainedExecutable { public void setMapReduceWaitTime(long t) { addExtraInfo(MAP_REDUCE_WAIT_TIME, t + ""); } - + public void setAlgorithm(AlgorithmEnum alg) { addExtraInfo("algorithm", alg.name()); } - + public AlgorithmEnum getAlgorithm() { String alg = getExtraInfo().get("algorithm"); return alg == null ? null : AlgorithmEnum.valueOf(alg); @@ -187,11 +221,11 @@ public class CubingJob extends DefaultChainedExecutable { public boolean isLayerCubing() { return AlgorithmEnum.LAYER == getAlgorithm(); } - + public boolean isInMemCubing() { return AlgorithmEnum.INMEM == getAlgorithm(); } - + public long findSourceRecordCount() { return Long.parseLong(findExtraInfo(SOURCE_RECORD_COUNT, "0")); } @@ -204,7 +238,7 @@ public class CubingJob extends DefaultChainedExecutable { // look for the info BACKWARD, let the last step that claims the cube size win return Long.parseLong(findExtraInfoBackward(CUBE_SIZE_BYTES, "0")); } - + public String findExtraInfo(String key, String dft) { return findExtraInfo(key, dft, false); } @@ -212,14 +246,14 @@ public class CubingJob extends DefaultChainedExecutable { public String findExtraInfoBackward(String key, String dft) { return findExtraInfo(key, dft, true); } - + private String findExtraInfo(String key, String dft, boolean backward) { ArrayList tasks = new ArrayList(getTasks()); - + if (backward) { Collections.reverse(tasks); } - + for (AbstractExecutable child : tasks) { Output output = executableManager.getOutput(child.getId()); String value = output.getExtra().get(key); -- 2.5.4 (Apple Git-61)