diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh index 6311584..460709e 100755 --- a/dev-support/test-patch.d/checkstyle.sh +++ b/dev-support/test-patch.d/checkstyle.sh @@ -29,39 +29,8 @@ function checkstyle_filefilter fi } -function checkstyle_mvnrunner -{ - local logfile=$1 - local output=$2 - local tmp=${PATCH_DIR}/$$.${RANDOM} - local j - - "${MVN}" clean test checkstyle:checkstyle -DskipTests \ - -Dcheckstyle.consoleOutput=true \ - "-D${PROJECT_NAME}PatchProcess" 2>&1 \ - | tee "${logfile}" \ - | ${GREP} ^/ \ - | ${SED} -e "s,${BASEDIR},.,g" \ - > "${tmp}" - - # the checkstyle output files are massive, so - # let's reduce the work by filtering out files - # that weren't changed. Some modules are - # MASSIVE and this can cut the output down to - # by orders of magnitude!! - for j in ${CHANGED_FILES}; do - ${GREP} "${j}" "${tmp}" >> "${output}" - done - - rm "${tmp}" 2>/dev/null -} - function checkstyle_preapply { - local module_suffix - local modules=${CHANGED_MODULES} - local module - verify_needed_test checkstyle if [[ $? == 0 ]]; then @@ -71,78 +40,23 @@ function checkstyle_preapply big_console_header "checkstyle plugin: prepatch" start_clock + echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}checkstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess" + if [[ $? != 0 ]] ; then + echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?" + add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} checkstyle compilation may be broken." + return 1 + fi - for module in ${modules} - do - pushd "${module}" >/dev/null - echo " Running checkstyle in ${module}" - module_suffix=$(basename "${module}") - - checkstyle_mvnrunner \ - "${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \ - "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" - - if [[ $? != 0 ]] ; then - echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?" - add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken." - fi - popd >/dev/null - done + cp -p "${BASEDIR}/target/checkstyle-result.xml" \ + "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" # keep track of how much as elapsed for us already CHECKSTYLE_TIMER=$(stop_clock) return 0 } -function checkstyle_calcdiffs -{ - local orig=$1 - local new=$2 - local diffout=$3 - local tmp=${PATCH_DIR}/cs.$$.${RANDOM} - local count=0 - local j - - # first, pull out just the errors - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch" - - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch" - - # compare the errors, generating a string of line - # numbers. Sorry portability: GNU diff makes this too easy - ${DIFF} --unchanged-line-format="" \ - --old-line-format="" \ - --new-line-format="%dn " \ - "${tmp}.branch" \ - "${tmp}.patch" > "${tmp}.lined" - - # now, pull out those lines of the raw output - # shellcheck disable=SC2013 - for j in $(cat "${tmp}.lined"); do - # shellcheck disable=SC2086 - head -${j} "${new}" | tail -1 >> "${diffout}" - done - - if [[ -f "${diffout}" ]]; then - # shellcheck disable=SC2016 - count=$(wc -l "${diffout}" | ${AWK} '{print $1}' ) - fi - rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null - echo "${count}" -} - function checkstyle_postapply { - local rc=0 - local module - local modules=${CHANGED_MODULES} - local module_suffix - local numprepatch=0 - local numpostpatch=0 - local diffpostpatch=0 - verify_needed_test checkstyle if [[ $? == 0 ]]; then @@ -157,49 +71,79 @@ function checkstyle_postapply # by setting the clock back offset_clock "${CHECKSTYLE_TIMER}" - for module in ${modules} - do - pushd "${module}" >/dev/null - echo " Running checkstyle in ${module}" - module_suffix=$(basename "${module}") - - checkstyle_mvnrunner \ - "${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \ - "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" - - if [[ $? != 0 ]] ; then - ((rc = rc +1)) - echo "Post-patch checkstyle compilation is broken." - add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken." - continue - fi - - #shellcheck disable=SC2016 - diffpostpatch=$(checkstyle_calcdiffs \ - "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \ - "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \ - "${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" ) - - if [[ ${diffpostpatch} -gt 0 ]] ; then - ((rc = rc + 1)) - - # shellcheck disable=SC2016 - numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}') - # shellcheck disable=SC2016 - numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}') - - add_jira_table -1 checkstyle "The applied patch generated "\ - "${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})." - footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt" - fi - - popd >/dev/null - done - - if [[ ${rc} -gt 0 ]] ; then - add_jira_footer checkstyle "${footer}" + echo_and_redirect "${PATCH_DIR}/patchcheckstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess" + if [[ $? != 0 ]] ; then + echo "Post-patch checkstyle compilation is broken." + add_jira_table -1 checkstyle "Post-patch checkstyle compilation is broken." + return 1 + fi + + cp -p "${BASEDIR}/target/checkstyle-result.xml" \ + "${PATCH_DIR}/checkstyle-result-patch.xml" + + checkstyle_runcomparison + + # shellcheck disable=SC2016 + CHECKSTYLE_POSTPATCH=$(wc -l "${PATCH_DIR}/checkstyle-result-diff.txt" | ${AWK} '{print $1}') + + if [[ ${CHECKSTYLE_POSTPATCH} -gt 0 ]] ; then + + add_jira_table -1 checkstyle "The applied patch generated "\ + "${CHECKSTYLE_POSTPATCH}" \ + " additional checkstyle issues." + add_jira_footer checkstyle "@@BASE@@/checkstyle-result-diff.txt" + return 1 fi add_jira_table +1 checkstyle "There were no new checkstyle issues." return 0 -} \ No newline at end of file +} + + +function checkstyle_runcomparison +{ + + python <(cat < master_dict[k]: + print_row(k, master_dict[k], child_errors) + +EOF +) "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" "${PATCH_DIR}/checkstyle-result-patch.xml" > "${PATCH_DIR}/checkstyle-result-diff.txt" + +} diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh index 5f38b6a..c1084a2 100755 --- a/dev-support/test-patch.d/shellcheck.sh +++ b/dev-support/test-patch.d/shellcheck.sh @@ -87,45 +87,6 @@ function shellcheck_preapply return 0 } -function shellcheck_calcdiffs -{ - local orig=$1 - local new=$2 - local diffout=$3 - local tmp=${PATCH_DIR}/sc.$$.${RANDOM} - local count=0 - local j - - # first, pull out just the errors - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch" - - # shellcheck disable=SC2016 - ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch" - - # compare the errors, generating a string of line - # numbers. Sorry portability: GNU diff makes this too easy - ${DIFF} --unchanged-line-format="" \ - --old-line-format="" \ - --new-line-format="%dn " \ - "${tmp}.branch" \ - "${tmp}.patch" > "${tmp}.lined" - - # now, pull out those lines of the raw output - # shellcheck disable=SC2013 - for j in $(cat "${tmp}.lined"); do - # shellcheck disable=SC2086 - head -${j} "${new}" | tail -1 >> "${diffout}" - done - - if [[ -f "${diffout}" ]]; then - # shellcheck disable=SC2016 - count=$(wc -l "${diffout}" | ${AWK} '{print $1}' ) - fi - rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null - echo "${count}" -} - function shellcheck_postapply { local i @@ -160,13 +121,16 @@ function shellcheck_postapply # shellcheck disable=SC2016 numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}') - diffPostpatch=$(shellcheck_calcdiffs \ - "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \ + ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \ "${PATCH_DIR}/patchshellcheck-result.txt" \ - "${PATCH_DIR}/diffpatchshellcheck.txt" - ) + | ${GREP} '^+\.' \ + > "${PATCH_DIR}/diffpatchshellcheck.txt" + + # shellcheck disable=SC2016 + diffPostpatch=$(wc -l "${PATCH_DIR}/diffpatchshellcheck.txt" | ${AWK} '{print $1}') - if [[ ${diffPostpatch} -gt 0 ]] ; then + if [[ ${diffPostpatch} -gt 0 + && ${numPostpatch} -gt ${numPrepatch} ]] ; then add_jira_table -1 shellcheck "The applied patch generated "\ "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})." add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt" diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh index 324481c..deac654 100755 --- a/dev-support/test-patch.d/whitespace.sh +++ b/dev-support/test-patch.d/whitespace.sh @@ -16,31 +16,25 @@ add_plugin whitespace -function whitespace_postapply +function whitespace_preapply { local count - local j big_console_header "Checking for whitespace at the end of lines" start_clock - pushd "${BASEDIR}" >/dev/null - for j in ${CHANGED_FILES}; do - ${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt" - done + ${GREP} '^+' "${PATCH_DIR}/patch" | ${GREP} '[[:blank:]]$' > "${PATCH_DIR}/whitespace.txt" # shellcheck disable=SC2016 count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}') if [[ ${count} -gt 0 ]]; then add_jira_table -1 whitespace "The patch has ${count}"\ - " line(s) that end in whitespace. Use git apply --whitespace=fix." + " line(s) that end in whitespace." add_jira_footer whitespace "@@BASE@@/whitespace.txt" - popd >/dev/null return 1 fi - popd >/dev/null add_jira_table +1 whitespace "The patch has no lines that end in whitespace." return 0 } diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index b6e1b03..ae21837 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -67,7 +67,7 @@ function setup_defaults EGREP=${EGREP:-/usr/xpg4/bin/egrep} GREP=${GREP:-/usr/xpg4/bin/grep} PATCH=${PATCH:-patch} - DIFF=${DIFF:-/usr/gnu/bin/diff} + DIFF=${DIFF:-diff} JIRACLI=${JIRA:-jira} ;; *) @@ -449,7 +449,7 @@ function verify_patchdir_still_exists if [[ ! -d ${PATCH_DIR} ]]; then rm "${commentfile}" 2>/dev/null - echo "(!) The patch artifact directory has been removed! " > "${commentfile}" + echo "(!) The patch artifact directory on has been removed! " > "${commentfile}" echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}" echo cat ${commentfile} @@ -468,49 +468,6 @@ function verify_patchdir_still_exists fi } -## @description generate a list of all files and line numbers that -## @description that were added/changed in the source repo -## @audience private -## @stability stable -## @params filename -## @replaceable no -function compute_gitdiff -{ - local outfile=$1 - local file - local line - local startline - local counter - local numlines - local actual - - pushd "${BASEDIR}" >/dev/null - while read line; do - if [[ ${line} =~ ^\+\+\+ ]]; then - file="./"$(echo "${line}" | cut -f2- -d/) - continue - elif [[ ${line} =~ ^@@ ]]; then - startline=$(echo "${line}" | cut -f3 -d' ' | cut -f1 -d, | tr -d + ) - numlines=$(echo "${line}" | cut -f3 -d' ' | cut -s -f2 -d, ) - # if this is empty, then just this line - # if it is 0, then no lines were added and this part of the patch - # is strictly a delete - if [[ ${numlines} == 0 ]]; then - continue - elif [[ -z ${numlines} ]]; then - numlines=1 - fi - counter=0 - until [[ ${counter} -gt ${numlines} ]]; do - ((actual=counter+startline)) - echo "${file}:${actual}:" >> "${outfile}" - ((counter=counter+1)) - done - fi - done < <("${GIT}" diff --unified=0 --no-color) - popd >/dev/null -} - ## @description Print the command to be executing to the screen. Then ## @description run the command, sending stdout and stderr to the given filename ## @description This will also ensure that any directories in ${BASEDIR} have @@ -524,7 +481,7 @@ function compute_gitdiff ## @returns $? function echo_and_redirect { - local logfile=$1 + logfile=$1 shift verify_patchdir_still_exists @@ -565,7 +522,7 @@ function hadoop_usage echo "Shell binary overrides:" echo "--awk-cmd= The 'awk' command to use (default 'awk')" - echo "--diff-cmd= The GNU-compatible 'diff' command to use (default 'diff')" + echo "--diff-cmd= The 'diff' command to use (default 'diff')" echo "--git-cmd= The 'git' command to use (default 'git')" echo "--grep-cmd= The 'grep' command to use (default 'grep')" echo "--mvn-cmd= The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')" @@ -628,10 +585,6 @@ function parse_args --grep-cmd=*) GREP=${i#*=} ;; - --help|-help|-h|help|--h|--\?|-\?|\?) - hadoop_usage - exit 0 - ;; --java-home) JAVA_HOME=${i#*=} ;; @@ -727,8 +680,6 @@ function parse_args cleanup_and_exit 1 fi fi - - GITDIFFLINES=${PATCH_DIR}/gitdifflines.txt } ## @description Locate the pom.xml file for a given directory @@ -765,14 +716,12 @@ function find_changed_files # get a list of all of the files that have been changed, # except for /dev/null (which would be present for new files). # Additionally, remove any a/ b/ patterns at the front - # of the patch filenames and any revision info at the end - # shellcheck disable=SC2016 + # of the patch filenames CHANGED_FILES=$(${GREP} -E '^(\+\+\+|---) ' "${PATCH_DIR}/patch" \ | ${SED} \ -e 's,^....,,' \ -e 's,^[ab]/,,' \ | ${GREP} -v /dev/null \ - | ${AWK} '{print $1}' \ | sort -u) } @@ -1603,7 +1552,7 @@ function check_javac > "${PATCH_DIR}/diffJavacWarnings.txt" add_jira_table -1 javac "The applied patch generated "\ - "$((patchJavacWarnings-${PATCH_BRANCH}JavacWarnings))" \ + "$((patchJavacWarnings-branchJavacWarnings))" \ " additional warning messages." add_jira_footer javac "@@BASE@@/diffJavacWarnings.txt" @@ -1763,7 +1712,6 @@ function check_findbugs "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \ "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" - #shellcheck disable=SC2016 newFindbugsWarnings=$("${FINDBUGS_HOME}/bin/filterBugs" \ -first "01/01/2000" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \ "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \ @@ -1939,12 +1887,10 @@ function check_unittests test_timeouts="${test_timeouts} ${module_test_timeouts}" result=1 fi - - #shellcheck disable=SC2026,SC2038,SC2016 + #shellcheck disable=SC2026,SC2038 module_failed_tests=$(find . -name 'TEST*.xml'\ | xargs "${GREP}" -l -E "= reservoir.length - min) { try { - if (stream == null) { - stream = new FileInputStream(new File(randomDevPath)); - } IOUtils.readFully(stream, reservoir, 0, reservoir.length); } catch (IOException e) { throw new RuntimeException("failed to fill reservoir", e); @@ -78,7 +75,21 @@ synchronized public void setConf(Configuration conf) { this.randomDevPath = conf.get( HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY, HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT); - close(); + File randomDevFile = new File(randomDevPath); + + try { + close(); + this.stream = new FileInputStream(randomDevFile); + } catch (IOException e) { + throw new RuntimeException(e); + } + + try { + fillReservoir(0); + } catch (RuntimeException e) { + close(); + throw e; + } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index eb568b9..e1bf6b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -872,6 +872,15 @@ private static void addDeprecatedKeys() { public static final String DEFAULT_NM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_NM_WEBAPP_HTTPS_PORT; + /** How often to monitor the node.*/ + public static final String NM_NODE_MON_INTERVAL_MS = NM_PREFIX + + "node-monitor.interval-ms"; + public final static int DEFAULT_NM_NODE_MON_INTERVAL_MS = 3000; + + /** Class that calculates nodes current resource utilization.*/ + public static final String NM_NODE_MON_RESOURCE_CALCULATOR = + NM_PREFIX + "node-monitor.resource-calculator.class"; + /** How often to monitor containers.*/ public final static String NM_CONTAINER_MON_INTERVAL_MS = NM_PREFIX + "container-monitor.interval-ms"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 6718b53..e1a533d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -121,7 +121,7 @@ protected NodeLabelsProvider createNodeLabelsProvider( } protected NodeResourceMonitor createNodeResourceMonitor() { - return new NodeResourceMonitorImpl(); + return new NodeResourceMonitorImpl(context); } protected ContainerManagerImpl createContainerManager(Context context, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java index be13d22..306dbc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java @@ -19,7 +19,15 @@ package org.apache.hadoop.yarn.server.nodemanager; import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.records.Resource; +/** + * Interface for monitoring the resources of a node. + */ public interface NodeResourceMonitor extends Service { - + /** + * Get the resource utilization of the node. + * @return resource utilization of the node. + */ + public Resource getUtilization(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java index ea82546..27777b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java @@ -18,13 +18,153 @@ package org.apache.hadoop.yarn.server.nodemanager; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; +import org.apache.hadoop.yarn.util.resource.Resources; +/** + * Implementation of the node resource monitor. It periodically tracks the + * resource utilization of the node and reports it to the NM. + */ public class NodeResourceMonitorImpl extends AbstractService implements NodeResourceMonitor { - public NodeResourceMonitorImpl() { + /** Logging infrastructure. */ + final static Log LOG = LogFactory + .getLog(NodeResourceMonitorImpl.class); + + /** Interval to monitor the node resource utilization. */ + private long monitoringInterval; + /** Thread to monitor the node resource utilization. */ + private MonitoringThread monitoringThread; + + /** Node manager context. */ + private final Context context; + /** Resource calculator. */ + private ResourceCalculatorPlugin resourceCalculatorPlugin; + + /** Current resource utilization of the node. */ + private Resource nodeUtilization; + + /** + * Initialize the node resource monitor. + * @param context Node manager context. + */ + public NodeResourceMonitorImpl(Context context) { super(NodeResourceMonitorImpl.class.getName()); + + this.context = context; + + this.monitoringThread = new MonitoringThread(); + } + + /** + * Initialize the service with the proper parameters. + */ + @Override + protected void serviceInit(Configuration conf) throws Exception { + this.monitoringInterval = + conf.getLong(YarnConfiguration.NM_NODE_MON_INTERVAL_MS, + YarnConfiguration.DEFAULT_NM_NODE_MON_INTERVAL_MS); + + Class clazz = + conf.getClass(YarnConfiguration.NM_NODE_MON_RESOURCE_CALCULATOR, null, + ResourceCalculatorPlugin.class); + + this.resourceCalculatorPlugin = + ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf); + + LOG.info(" Using ResourceCalculatorPlugin : " + + this.resourceCalculatorPlugin); } + /** + * Check if we should be monitoring. + * @return true if we can monitor the node resource utilization. + */ + private boolean isEnabled() { + if (resourceCalculatorPlugin == null) { + LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + + this.getClass().getName() + " is disabled."); + return false; + } + return true; + } + + /** + * Start the thread that does the node resource utilization monitoring. + */ + @Override + protected void serviceStart() throws Exception { + if (this.isEnabled()) { + this.monitoringThread.start(); + } + super.serviceStart(); + } + + /** + * Stop the thread that does the node resource utilization monitoring. + */ + @Override + protected void serviceStop() throws Exception { + if (this.isEnabled()) { + this.monitoringThread.interrupt(); + try { + this.monitoringThread.join(); + } catch (InterruptedException e) { + ; + } + } + super.serviceStop(); + } + + /** + * Thread that monitors the resource utilization of this node. + */ + private class MonitoringThread extends Thread { + /** + * Initialize the node resource monitoring thread. + */ + public MonitoringThread() { + super("Node Resource Monitor"); + } + + /** + * Periodically monitor the resource utilization of the node. + */ + @Override + public void run() { + while (true) { + // Get node utilization and save it into the health status + long memory = resourceCalculatorPlugin.getPhysicalMemorySize() - + resourceCalculatorPlugin.getAvailablePhysicalMemorySize(); + float cpu = resourceCalculatorPlugin.getCpuUsage(); + nodeUtilization = Resources.createResource( + (int)(memory >> 20), // B -> MB + (int)Math.ceil(cpu)); // CPU% -> VCores (1CPU at 100% is 1) + + try { + Thread.sleep(monitoringInterval); + } catch (InterruptedException e) { + LOG.warn(NodeResourceMonitorImpl.class.getName() + + " is interrupted. Exiting."); + break; + } + } + } + } + + /** + * Get the resource utilization of the node. + * @return resource utilization of the node. + */ + @Override + public Resource getUtilization() { + return this.nodeUtilization; + } } diff --git a/pom.xml b/pom.xml index 3bad969..3f2aafc 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs hadoop-assemblies hadoop-maven-plugins hadoop-common-project - hadoop-hdfs-project + hadoop-yarn-project hadoop-mapreduce-project hadoop-tools