Index: hadoop-common-project/hadoop-common/src/main/native/native.vcxproj =================================================================== --- hadoop-common-project/hadoop-common/src/main/native/native.vcxproj (revision 1604031) +++ hadoop-common-project/hadoop-common/src/main/native/native.vcxproj (working copy) @@ -1,5 +1,4 @@ - - + - @@ -35,6 +33,7 @@ false true Unicode + v110 @@ -61,9 +60,7 @@ $(ZLIB_HOME);$(IncludePath) - + Index: hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj =================================================================== --- hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj (revision 1604031) +++ hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj (working copy) @@ -1,5 +1,4 @@  - - @@ -46,11 +44,13 @@ StaticLibrary true Unicode + v110 StaticLibrary true Unicode + v110 StaticLibrary @@ -57,6 +57,7 @@ false true Unicode + v110 StaticLibrary @@ -63,6 +64,7 @@ false true Unicode + v110 @@ -168,4 +170,4 @@ - + \ No newline at end of file Index: hadoop-common-project/hadoop-common/src/main/winutils/task.c =================================================================== --- hadoop-common-project/hadoop-common/src/main/winutils/task.c (revision 1604031) +++ hadoop-common-project/hadoop-common/src/main/winutils/task.c (working copy) @@ -78,7 +78,7 @@ } } - if (argc == 4) { + if (argc == 4 || argc == 6) { if (wcscmp(argv[1], L"create") == 0) { *command = TaskCreate; @@ -101,6 +101,21 @@ // GetLastError: otherwise DWORD createTask(__in PCWSTR jobObjName,__in PWSTR cmdLine) { + return createTaskWithLimit(jobObjName, cmdLine, -1, -1); +} + +//---------------------------------------------------------------------------- +// Function: createTask +// +// Description: +// Creates a task via a jobobject. Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +DWORD createTaskWithLimit(__in PCWSTR jobObjName, __in PWSTR cmdLine, __in long memory, __in long vcores) +{ DWORD err = ERROR_SUCCESS; DWORD exitCode = EXIT_FAILURE; STARTUPINFO si; @@ -119,6 +134,12 @@ return err; } jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (memory > 0) + { + jeli.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_JOB_MEMORY; + jeli.ProcessMemoryLimit = memory * 1024 * 1024; + jeli.JobMemoryLimit = memory * 1024 * 1024; + } if(SetInformationJobObject(jobObject, JobObjectExtendedLimitInformation, &jeli, @@ -128,6 +149,20 @@ CloseHandle(jobObject); return err; } + if (vcores > 0) + { + JOBOBJECT_CPU_RATE_CONTROL_INFORMATION jcrci = { 0 }; + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + jcrci.ControlFlags = JOB_OBJECT_CPU_RATE_CONTROL_ENABLE | JOB_OBJECT_CPU_RATE_CONTROL_HARD_CAP; + jcrci.CpuRate = vcores * (10000 / sysinfo.dwNumberOfProcessors); + if(SetInformationJobObject(jobObject, JobObjectCpuRateControlInformation, &jcrci, sizeof(jcrci)) == 0) + { + err = GetLastError(); + CloseHandle(jobObject); + return err; + } + } if(AssignProcessToJobObject(jobObject, GetCurrentProcess()) == 0) { @@ -385,7 +420,40 @@ { // Create the task jobobject // - dwErrorCode = createTask(argv[2], argv[3]); + if (argc == 4) + { + dwErrorCode = createTask(argv[2], argv[3]); + } + else if (argc == 6) + { + long memory = -1; + long vcores = -1; + wchar_t *end; + memory = wcstol(argv[2], &end, 10); + if (*end != '\0') + { + dwErrorCode = ERROR_INVALID_COMMAND_LINE; + fwprintf(stderr, L"Incorrect memory limit setting.\n\n"); + TaskUsage(); + goto TaskExit; + } + vcores = wcstol(argv[3], &end, 10); + if (*end != '\0') + { + dwErrorCode = ERROR_INVALID_COMMAND_LINE; + fwprintf(stderr, L"Incorrect cpu limit setting.\n\n"); + TaskUsage(); + goto TaskExit; + } + dwErrorCode = createTaskWithLimit(argv[4], argv[5], memory, vcores); + } + else + { + dwErrorCode = ERROR_INVALID_COMMAND_LINE; + fwprintf(stderr, L"Incorrect command line arguments.\n\n"); + TaskUsage(); + goto TaskExit; + } if (dwErrorCode != ERROR_SUCCESS) { ReportErrorCode(L"createTask", dwErrorCode); @@ -453,10 +521,12 @@ // ProcessTree.isSetsidSupported() fwprintf(stdout, L"\ Usage: task create [TASKNAME] [COMMAND_LINE] |\n\ + task create [CPU] [MEMORY] [TASKNAME] [COMMAND_LINE] |\n\ task isAlive [TASKNAME] |\n\ task kill [TASKNAME]\n\ task processList [TASKNAME]\n\ Creates a new task jobobject with taskname\n\ + Creates a new task jobobject with taskname, CPU, and memory limits\n\ Checks if task jobobject is alive\n\ Kills task jobobject\n\ Prints to stdout a list of processes in the task\n\ Index: hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln =================================================================== --- hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln (revision 1604284) +++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln (working copy) @@ -1,22 +1,6 @@  -Microsoft Visual Studio Solution File, Format Version 11.00 -# Visual Studio 2010 - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2012 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winutils", "winutils.vcxproj", "{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}" ProjectSection(ProjectDependencies) = postProject {12131AA7-902E-4A6D-9CE3-043261D22A12} = {12131AA7-902E-4A6D-9CE3-043261D22A12} @@ -34,8 +18,8 @@ GlobalSection(ProjectConfigurationPlatforms) = postSolution {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.ActiveCfg = Debug|x64 {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.Build.0 = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Debug|x64 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Release|x64 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Release|x64 {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.ActiveCfg = Release|Win32 {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.Build.0 = Release|Win32 {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.ActiveCfg = Release|x64 @@ -42,8 +26,8 @@ {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.Build.0 = Release|x64 {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.ActiveCfg = Debug|x64 {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.Build.0 = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Debug|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Release|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Release|x64 {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.ActiveCfg = Release|Win32 {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.Build.0 = Release|Win32 {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.ActiveCfg = Release|x64 Index: hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj =================================================================== --- hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj (revision 1604031) +++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj (working copy) @@ -1,5 +1,4 @@  - - @@ -46,11 +44,13 @@ Application true Unicode + v110 Application true Unicode + v110 Application @@ -57,6 +57,7 @@ false true Unicode + v110 Application @@ -63,6 +64,7 @@ false true Unicode + v110 Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (revision 1604031) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (working copy) @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; @@ -188,6 +189,11 @@ readLock.unlock(); } } + + protected String[] getRunCommand(String command, String groupId, + Configuration conf) { + return getRunCommand(command, groupId, conf, null); + } /** * Return a command to execute the given command in OS shell. @@ -195,8 +201,8 @@ * and associate the given groupId in a process group. On * non-Windows, groupId is ignored. */ - protected static String[] getRunCommand(String command, String groupId, - Configuration conf) { + protected String[] getRunCommand(String command, String groupId, + Configuration conf, Resource resource) { boolean containerSchedPriorityIsSet = false; int containerSchedPriorityAdjustment = YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY; Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (revision 1604031) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (working copy) @@ -184,7 +184,7 @@ // Setup command to run String[] command = getRunCommand(sb.getWrapperScriptPath().toString(), - containerIdStr, this.getConf()); + containerIdStr, this.getConf(), container.getResource()); LOG.info("launchContainer: " + Arrays.toString(command)); shExec = new ShellCommandExecutor( Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java =================================================================== --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java (revision 1604031) +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java (working copy) @@ -31,7 +31,8 @@ @Test (timeout = 5000) public void testRunCommandNoPriority() throws Exception { Configuration conf = new Configuration(); - String[] command = ContainerExecutor.getRunCommand("echo", "group1", conf); + ContainerExecutor exe = new DefaultContainerExecutor(); + String[] command = exe.getRunCommand("echo", "group1", conf); assertTrue("first command should be the run command for the platform", command[0].equals(Shell.WINUTILS) || command[0].equals("bash")); } @@ -40,7 +41,8 @@ public void testRunCommandwithPriority() throws Exception { Configuration conf = new Configuration(); conf.setInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 2); - String[] command = ContainerExecutor.getRunCommand("echo", "group1", conf); + ContainerExecutor exe = new DefaultContainerExecutor(); + String[] command = exe.getRunCommand("echo", "group1", conf); if (Shell.WINDOWS) { // windows doesn't currently support assertEquals("first command should be the run command for the platform", @@ -54,7 +56,7 @@ // test with negative number conf.setInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, -5); - command = ContainerExecutor.getRunCommand("echo", "group1", conf); + command = exe.getRunCommand("echo", "group1", conf); if (Shell.WINDOWS) { // windows doesn't currently support assertEquals("first command should be the run command for the platform",