Index: hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
===================================================================
--- hadoop-common-project/hadoop-common/src/main/native/native.vcxproj (revision 1604031)
+++ hadoop-common-project/hadoop-common/src/main/native/native.vcxproj (working copy)
@@ -1,5 +1,4 @@
-
-
+
-
@@ -35,6 +33,7 @@
false
true
Unicode
+ v110
@@ -61,9 +60,7 @@
$(ZLIB_HOME);$(IncludePath)
-
+
Index: hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
===================================================================
--- hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj (revision 1604031)
+++ hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj (working copy)
@@ -1,5 +1,4 @@
-
-
@@ -46,11 +44,13 @@
StaticLibrary
true
Unicode
+ v110
StaticLibrary
true
Unicode
+ v110
StaticLibrary
@@ -57,6 +57,7 @@
false
true
Unicode
+ v110
StaticLibrary
@@ -63,6 +64,7 @@
false
true
Unicode
+ v110
@@ -168,4 +170,4 @@
-
+
\ No newline at end of file
Index: hadoop-common-project/hadoop-common/src/main/winutils/task.c
===================================================================
--- hadoop-common-project/hadoop-common/src/main/winutils/task.c (revision 1604031)
+++ hadoop-common-project/hadoop-common/src/main/winutils/task.c (working copy)
@@ -78,7 +78,7 @@
}
}
- if (argc == 4) {
+ if (argc == 4 || argc == 6) {
if (wcscmp(argv[1], L"create") == 0)
{
*command = TaskCreate;
@@ -101,6 +101,21 @@
// GetLastError: otherwise
DWORD createTask(__in PCWSTR jobObjName,__in PWSTR cmdLine)
{
+ return createTaskWithLimit(jobObjName, cmdLine, -1, -1);
+}
+
+//----------------------------------------------------------------------------
+// Function: createTask
+//
+// Description:
+// Creates a task via a jobobject. Outputs the
+// appropriate information to stdout on success, or stderr on failure.
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+DWORD createTaskWithLimit(__in PCWSTR jobObjName, __in PWSTR cmdLine, __in long memory, __in long vcores)
+{
DWORD err = ERROR_SUCCESS;
DWORD exitCode = EXIT_FAILURE;
STARTUPINFO si;
@@ -119,6 +134,12 @@
return err;
}
jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+ if (memory > 0)
+ {
+ jeli.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_JOB_MEMORY;
+ jeli.ProcessMemoryLimit = memory * 1024 * 1024;
+ jeli.JobMemoryLimit = memory * 1024 * 1024;
+ }
if(SetInformationJobObject(jobObject,
JobObjectExtendedLimitInformation,
&jeli,
@@ -128,6 +149,20 @@
CloseHandle(jobObject);
return err;
}
+ if (vcores > 0)
+ {
+ JOBOBJECT_CPU_RATE_CONTROL_INFORMATION jcrci = { 0 };
+ SYSTEM_INFO sysinfo;
+ GetSystemInfo(&sysinfo);
+ jcrci.ControlFlags = JOB_OBJECT_CPU_RATE_CONTROL_ENABLE | JOB_OBJECT_CPU_RATE_CONTROL_HARD_CAP;
+ jcrci.CpuRate = vcores * (10000 / sysinfo.dwNumberOfProcessors);
+ if(SetInformationJobObject(jobObject, JobObjectCpuRateControlInformation, &jcrci, sizeof(jcrci)) == 0)
+ {
+ err = GetLastError();
+ CloseHandle(jobObject);
+ return err;
+ }
+ }
if(AssignProcessToJobObject(jobObject, GetCurrentProcess()) == 0)
{
@@ -385,7 +420,40 @@
{
// Create the task jobobject
//
- dwErrorCode = createTask(argv[2], argv[3]);
+ if (argc == 4)
+ {
+ dwErrorCode = createTask(argv[2], argv[3]);
+ }
+ else if (argc == 6)
+ {
+ long memory = -1;
+ long vcores = -1;
+ wchar_t *end;
+ memory = wcstol(argv[2], &end, 10);
+ if (*end != '\0')
+ {
+ dwErrorCode = ERROR_INVALID_COMMAND_LINE;
+ fwprintf(stderr, L"Incorrect memory limit setting.\n\n");
+ TaskUsage();
+ goto TaskExit;
+ }
+ vcores = wcstol(argv[3], &end, 10);
+ if (*end != '\0')
+ {
+ dwErrorCode = ERROR_INVALID_COMMAND_LINE;
+ fwprintf(stderr, L"Incorrect cpu limit setting.\n\n");
+ TaskUsage();
+ goto TaskExit;
+ }
+ dwErrorCode = createTaskWithLimit(argv[4], argv[5], memory, vcores);
+ }
+ else
+ {
+ dwErrorCode = ERROR_INVALID_COMMAND_LINE;
+ fwprintf(stderr, L"Incorrect command line arguments.\n\n");
+ TaskUsage();
+ goto TaskExit;
+ }
if (dwErrorCode != ERROR_SUCCESS)
{
ReportErrorCode(L"createTask", dwErrorCode);
@@ -453,10 +521,12 @@
// ProcessTree.isSetsidSupported()
fwprintf(stdout, L"\
Usage: task create [TASKNAME] [COMMAND_LINE] |\n\
+ task create [CPU] [MEMORY] [TASKNAME] [COMMAND_LINE] |\n\
task isAlive [TASKNAME] |\n\
task kill [TASKNAME]\n\
task processList [TASKNAME]\n\
Creates a new task jobobject with taskname\n\
+ Creates a new task jobobject with taskname, CPU, and memory limits\n\
Checks if task jobobject is alive\n\
Kills task jobobject\n\
Prints to stdout a list of processes in the task\n\
Index: hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
===================================================================
--- hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln (revision 1604284)
+++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln (working copy)
@@ -1,22 +1,6 @@
-Microsoft Visual Studio Solution File, Format Version 11.00
-# Visual Studio 2010
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2012
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winutils", "winutils.vcxproj", "{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}"
ProjectSection(ProjectDependencies) = postProject
{12131AA7-902E-4A6D-9CE3-043261D22A12} = {12131AA7-902E-4A6D-9CE3-043261D22A12}
@@ -34,8 +18,8 @@
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.ActiveCfg = Debug|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.Build.0 = Debug|x64
- {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Debug|x64
- {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Debug|x64
+ {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Release|x64
+ {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Release|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.ActiveCfg = Release|Win32
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.Build.0 = Release|Win32
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.ActiveCfg = Release|x64
@@ -42,8 +26,8 @@
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.Build.0 = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.ActiveCfg = Debug|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.Build.0 = Debug|x64
- {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Debug|x64
- {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Debug|x64
+ {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Release|x64
+ {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.ActiveCfg = Release|Win32
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.Build.0 = Release|Win32
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.ActiveCfg = Release|x64
Index: hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
===================================================================
--- hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj (revision 1604031)
+++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj (working copy)
@@ -1,5 +1,4 @@
-
-
@@ -46,11 +44,13 @@
Application
true
Unicode
+ v110
Application
true
Unicode
+ v110
Application
@@ -57,6 +57,7 @@
false
true
Unicode
+ v110
Application
@@ -63,6 +64,7 @@
false
true
Unicode
+ v110
Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
===================================================================
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (revision 1604031)
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (working copy)
@@ -37,6 +37,7 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
@@ -188,6 +189,11 @@
readLock.unlock();
}
}
+
+ protected String[] getRunCommand(String command, String groupId,
+ Configuration conf) {
+ return getRunCommand(command, groupId, conf, null);
+ }
/**
* Return a command to execute the given command in OS shell.
@@ -195,8 +201,8 @@
* and associate the given groupId in a process group. On
* non-Windows, groupId is ignored.
*/
- protected static String[] getRunCommand(String command, String groupId,
- Configuration conf) {
+ protected String[] getRunCommand(String command, String groupId,
+ Configuration conf, Resource resource) {
boolean containerSchedPriorityIsSet = false;
int containerSchedPriorityAdjustment =
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;
Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
===================================================================
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (revision 1604031)
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (working copy)
@@ -184,7 +184,7 @@
// Setup command to run
String[] command = getRunCommand(sb.getWrapperScriptPath().toString(),
- containerIdStr, this.getConf());
+ containerIdStr, this.getConf(), container.getResource());
LOG.info("launchContainer: " + Arrays.toString(command));
shExec = new ShellCommandExecutor(
Index: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java
===================================================================
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java (revision 1604031)
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java (working copy)
@@ -31,7 +31,8 @@
@Test (timeout = 5000)
public void testRunCommandNoPriority() throws Exception {
Configuration conf = new Configuration();
- String[] command = ContainerExecutor.getRunCommand("echo", "group1", conf);
+ ContainerExecutor exe = new DefaultContainerExecutor();
+ String[] command = exe.getRunCommand("echo", "group1", conf);
assertTrue("first command should be the run command for the platform",
command[0].equals(Shell.WINUTILS) || command[0].equals("bash"));
}
@@ -40,7 +41,8 @@
public void testRunCommandwithPriority() throws Exception {
Configuration conf = new Configuration();
conf.setInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 2);
- String[] command = ContainerExecutor.getRunCommand("echo", "group1", conf);
+ ContainerExecutor exe = new DefaultContainerExecutor();
+ String[] command = exe.getRunCommand("echo", "group1", conf);
if (Shell.WINDOWS) {
// windows doesn't currently support
assertEquals("first command should be the run command for the platform",
@@ -54,7 +56,7 @@
// test with negative number
conf.setInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, -5);
- command = ContainerExecutor.getRunCommand("echo", "group1", conf);
+ command = exe.getRunCommand("echo", "group1", conf);
if (Shell.WINDOWS) {
// windows doesn't currently support
assertEquals("first command should be the run command for the platform",