diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index 170704f..68a1db2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; +import com.google.common.annotations.VisibleForTesting; + /** * Windows secure container executor (WSCE). * This class offers a secure container executor on Windows, similar to the @@ -67,13 +69,52 @@ private static final Log LOG = LogFactory .getLog(WindowsSecureContainerExecutor.class); + @VisibleForTesting public static final String LOCALIZER_PID_FORMAT = "STAR_LOCALIZER_%s"; + @VisibleForTesting + public static final String WIN_DEV_NULL = "nul:"; + + /** + * This interface is solely used for purpose of allowing IoC and mocking + * in WSCE unit tests. + * + */ + @VisibleForTesting + public interface NativeAPI { + + void killTask(String pid) throws IOException; + + void mkdir(Path path) throws IOException; + + void chmod(Path p, short short1) throws IOException; + + void chown(Path p, String username, String groupname) throws IOException; + + OutputStream create(Path f, boolean append) throws IOException; + + boolean deleteFile(Path p) throws IOException; + + boolean deleteDirectory(Path p) throws IOException; + + void copy(Path src, Path dst, boolean b) throws IOException; + + void move(Path classPathJar, Path dst, boolean b) throws IOException; + + Native.WinutilsProcessStub createTaskAsUser( + String cwd, String jobName, String userName, + String pidFile, String cmdLine) throws IOException; + + } + + @VisibleForTesting + public static NativeAPI _nativeAPI; /** * This class is a container for the JNI Win32 native methods used by WSCE. */ - private static class Native { + @VisibleForTesting + public static class Native { private static boolean nativeLoaded = false; @@ -322,7 +363,7 @@ protected boolean mkOneDir(File p2f) throws IOException { // File.mkdir returns false, does not throw. Must mimic it. try { - Native.Elevated.mkdir(path); + _nativeAPI.mkdir(path); ret = true; } catch(Throwable e) { @@ -340,7 +381,7 @@ public void setPermission(Path p, FsPermission permission) if (LOG.isDebugEnabled()) { LOG.debug(String.format("EFS:setPermission: %s %s", p, permission)); } - Native.Elevated.chmod(p, permission.toShort()); + _nativeAPI.chmod(p, permission.toShort()); } @Override @@ -350,7 +391,7 @@ public void setOwner(Path p, String username, String groupname) LOG.debug(String.format("EFS:setOwner: %s %s %s", p, username, groupname)); } - Native.Elevated.chown(p, username, groupname); + _nativeAPI.chown(p, username, groupname); } @Override @@ -359,7 +400,7 @@ protected OutputStream createOutputStream(Path f, boolean append) if (LOG.isDebugEnabled()) { LOG.debug(String.format("EFS:create: %s %b", f, append)); } - return Native.Elevated.create(f, append); + return _nativeAPI.create(f, append); } @Override @@ -378,7 +419,7 @@ public boolean delete(Path p, boolean recursive) throws IOException { return false; } else if (f.isFile()) { - return Native.Elevated.deleteFile(p); + return _nativeAPI.deleteFile(p); } else if (f.isDirectory()) { @@ -396,7 +437,7 @@ else if (f.isDirectory()) { } } if (childCount == 0) { - return Native.Elevated.deleteDirectory(p); + return _nativeAPI.deleteDirectory(p); } else { throw new IOException("Directory " + f.toString() + " is not empty"); @@ -420,7 +461,8 @@ protected ElevatedFileSystem() throws IOException, URISyntaxException { } } - private static class WintuilsProcessStubExecutor + @VisibleForTesting + public static class WintuilsProcessStubExecutor implements Shell.CommandExecutor { private Native.WinutilsProcessStub processStub; private StringBuilder output = new StringBuilder(); @@ -507,7 +549,7 @@ public void execute() throws IOException { if (state != State.INIT) { throw new IOException("Process is already started"); } - processStub = Native.createTaskAsUser(cwd, + processStub = _nativeAPI.createTaskAsUser(cwd, jobName, userName, pidFile, cmdLine); state = State.RUNNING; @@ -538,10 +580,72 @@ public void close() { private String nodeManagerGroup; + + static { + _nativeAPI = new NativeAPI() { + + @Override + public void killTask(String pid) throws IOException { + Native.Elevated.killTask(pid); + } + + @Override + public void mkdir(Path path) throws IOException { + Native.Elevated.mkdir(path); + } + + @Override + public void chmod(Path path, short mode) throws IOException { + Native.Elevated.chmod(path, mode); + } + + @Override + public void chown(Path path, String user, String group) + throws IOException { + Native.Elevated.chown(path, user, group); + } + + @Override + public OutputStream create(Path path, boolean append) throws IOException { + return Native.Elevated.create(path, append); + } + + @Override + public boolean deleteFile(Path path) throws IOException { + return Native.Elevated.deleteFile(path); + } + + @Override + public boolean deleteDirectory(Path path) throws IOException { + return Native.Elevated.deleteDirectory(path); + } + + @Override + public void copy(Path src, Path dst, boolean replace) throws IOException { + Native.Elevated.copy(src, dst, replace); + } + + @Override + public void move(Path src, Path dst, boolean replace) throws IOException { + Native.Elevated.move(src, dst, replace); + } + + @Override + public Native.WinutilsProcessStub createTaskAsUser( + String cwd, String jobName, String user, + String pidFile, String cmdLine) + throws IOException { + return Native.createTaskAsUser(cwd, jobName, user, pidFile, cmdLine); + } + }; + } + + /** * Permissions for user WSCE dirs. */ - static final short DIR_PERM = (short)0750; + @VisibleForTesting + public static final short DIR_PERM = (short)0750; public WindowsSecureContainerExecutor() throws IOException, URISyntaxException { @@ -580,8 +684,8 @@ protected void copyFile(Path src, Path dst, String owner) throws IOException { LOG.debug(String.format("copyFile: %s -> %s owner:%s", src.toString(), dst.toString(), owner)); } - Native.Elevated.copy(src, dst, true); - Native.Elevated.chown(dst, owner, nodeManagerGroup); + _nativeAPI.copy(src, dst, true); + _nativeAPI.chown(dst, owner, nodeManagerGroup); } @Override @@ -598,7 +702,7 @@ protected void createDir(Path dirPath, FsPermission perms, } super.createDir(dirPath, perms, createParent, owner); - lfs.setOwner(dirPath, owner, nodeManagerGroup); + _nativeAPI.chown(dirPath, owner, nodeManagerGroup); } @Override @@ -608,8 +712,9 @@ protected void setScriptExecutable(Path script, String owner) LOG.debug(String.format("setScriptExecutable: %s owner:%s", script.toString(), owner)); } - super.setScriptExecutable(script, owner); - Native.Elevated.chown(script, owner, nodeManagerGroup); + _nativeAPI.chmod(script, + ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION.toShort()); + _nativeAPI.chown(script, owner, nodeManagerGroup); } @Override @@ -622,8 +727,8 @@ public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner) createDir(pwd, new FsPermission(DIR_PERM), true, owner); String fileName = classPathJar.getName(); Path dst = new Path(pwd, fileName); - Native.Elevated.move(classPathJar, dst, true); - Native.Elevated.chown(dst, owner, nodeManagerGroup); + _nativeAPI.move(classPathJar, dst, true); + _nativeAPI.chown(dst, owner, nodeManagerGroup); return dst; } @@ -693,7 +798,7 @@ public void startLocalizer(Path nmPrivateContainerTokens, WintuilsProcessStubExecutor stubExecutor = new WintuilsProcessStubExecutor( cwdApp.getAbsolutePath(), - localizerPid, user, "nul:", cmdLine); + localizerPid, user, WIN_DEV_NULL, cmdLine); try { stubExecutor.execute(); stubExecutor.validateResult(); @@ -727,7 +832,7 @@ protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, @Override protected void killContainer(String pid, Signal signal) throws IOException { - Native.Elevated.killTask(pid); + _nativeAPI.killTask(pid); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestWindowsSecureContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestWindowsSecureContainerExecutor.java new file mode 100644 index 0000000..6cc5e27 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestWindowsSecureContainerExecutor.java @@ -0,0 +1,352 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.nodemanager; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.times; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor.Native.WinutilsProcessStub; +import org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor.NativeAPI; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; +import org.hamcrest.CoreMatchers; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +/** + * Unit tests for Windows Secure Container Executor + * + */ +public class TestWindowsSecureContainerExecutor { + + private static final Log LOG = LogFactory + .getLog(TestWindowsSecureContainerExecutor.class); + + private WindowsSecureContainerExecutor testedExecutor = null; + + private LocalDirsHandlerService dirsHandler; + + private NativeAPI mockAPI; + + private WinutilsProcessStub mockStub; + + private Container mockContainer; + + private ContainerId mockContainerId; + + private ApplicationAttemptId mockApplicationAttemptId; + + private ApplicationId mockApplicationId; + + private ContainerLaunchContext mockLaunchContext; + + private static final String yarnGroup = "yarn"; + + public static class RegexMatcher extends TypeSafeMatcher { + + private final String regEx; + + public static RegexMatcher matchesRegex(String regEx) { + return new RegexMatcher(regEx); + } + + public RegexMatcher(String regEx) { + this.regEx = regEx; + } + + @Override + public void describeTo(Description description) { + description.appendText(String.format("matches regex `%s`", regEx)); + } + + @Override + protected boolean matchesSafely(String item) { + return item.matches(this.regEx); + } + } + + @Before + public void setup() throws IOException, URISyntaxException { + assumeTrue(Shell.WINDOWS); + + Configuration conf = new Configuration(); + + File absLocalDir = new File("target/test-dir", "local").getAbsoluteFile(); + absLocalDir.mkdirs(); + LOG.info(absLocalDir); + conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLocalDir.getAbsolutePath()); + + File absLogsDir = new File("target/test-dir", "logs").getAbsoluteFile(); + absLogsDir.mkdirs(); + LOG.info(absLogsDir); + conf.set(YarnConfiguration.NM_LOG_DIRS, absLogsDir.getAbsolutePath()); + + conf.set(YarnConfiguration.NM_WINDOWS_SECURE_CONTAINER_GROUP, yarnGroup); + + dirsHandler = new LocalDirsHandlerService(); + dirsHandler.init(conf); + + mockAPI = mock(WindowsSecureContainerExecutor.NativeAPI.class); + mockStub = mock( + WindowsSecureContainerExecutor.Native.WinutilsProcessStub.class); + + InputStream mockStream = mock(InputStream.class); + + when(mockStub.getInputStream()).thenReturn(mockStream); + when(mockStub.getErrorStream()).thenReturn(mockStream); + when(mockStub.exitValue()).thenReturn(0); + + when(mockAPI.createTaskAsUser(anyString(), anyString(), + anyString(), anyString(), anyString())).thenReturn(mockStub); + + mockApplicationId = mock(ApplicationId.class); + when(mockApplicationId.toString()).thenReturn("mock_app_id"); + + mockApplicationAttemptId = + mock(ApplicationAttemptId.class); + when(mockApplicationAttemptId.getApplicationId()). + thenReturn(mockApplicationId); + + mockContainerId = mock(ContainerId.class); + when(mockContainerId.getApplicationAttemptId()). + thenReturn(mockApplicationAttemptId); + when(mockContainerId.toString()).thenReturn("mock_container_id"); + + mockLaunchContext = mock(ContainerLaunchContext.class); + when(mockLaunchContext.getEnvironment()).thenReturn( + new HashMap()); + + mockContainer = mock(Container.class); + when(mockContainer.getContainerId()).thenReturn(mockContainerId); + when(mockContainer.getLaunchContext()).thenReturn(mockLaunchContext); + + WindowsSecureContainerExecutor._nativeAPI = mockAPI; + testedExecutor = new WindowsSecureContainerExecutor(); + testedExecutor.setConf(conf); + } + + @Test + public void testStartLocalizerCreateTaskAsUser() + throws IOException, InterruptedException { + + String appId = "APP_ID"; + String user = "nobody"; + String locId = "LOC_ID"; + Path nmPrivateContainerTokens = new Path("tokens"); + InetSocketAddress nmAddr = mock(InetSocketAddress.class); + + testedExecutor.startLocalizer(nmPrivateContainerTokens, + nmAddr, user, appId, locId, dirsHandler); + + ArgumentCaptor captorCwd = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captorJobName = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captorUserName = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captorPidFile = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captorCmdLine= ArgumentCaptor.forClass(String.class); + + verify(mockAPI).createTaskAsUser( + captorCwd.capture(), captorJobName.capture(), + captorUserName.capture(), captorPidFile.capture(), + captorCmdLine.capture()); + assertThat(captorCwd.getValue(), CoreMatchers.endsWith( + "test-dir\\local\\usercache\\" + user + "\\appcache\\" + appId)); + assertEquals( + String.format(WindowsSecureContainerExecutor.LOCALIZER_PID_FORMAT, locId), + captorJobName.getValue()); + assertEquals(user, captorUserName.getValue()); + assertEquals(WindowsSecureContainerExecutor.WIN_DEV_NULL, + captorPidFile.getValue()); + + String cmdLine = captorCmdLine.getValue(); + + assertThat(cmdLine, CoreMatchers.startsWith( + System.getProperty("java.home") + "\\bin\\java.exe")); + assertThat(cmdLine, CoreMatchers.containsString( + "-Djava.library.path=" + System.getProperty("java.library.path"))); + assertThat(cmdLine, RegexMatcher.matchesRegex( + ".+\\ -classpath\\ .+/test-dir/local/usercache/" + user + + "/appcache/" + appId + "/classpath-\\d+\\.jar\\ .+")); + assertThat(cmdLine, CoreMatchers.containsString( + ContainerLocalizer.class.getName())); + assertThat(cmdLine, CoreMatchers.endsWith("test-dir/local")); + } + + @Test + public void testStartLocalizerMkDirs() + throws IOException, InterruptedException { + + String appId = "APP_ID"; + String user = "nobody"; + String locId = "LOC_ID"; + Path nmPrivateContainerTokens = new Path("tokens"); + InetSocketAddress nmAddr = mock(InetSocketAddress.class); + + testedExecutor.startLocalizer(nmPrivateContainerTokens, + nmAddr, user, appId, locId, dirsHandler); + + ArgumentCaptor captureMkDir = ArgumentCaptor.forClass(Path.class); + + String expectedDirs[] = new String[] { + "test-dir/local/usercache", + "test-dir/local/usercache/" + user, + "test-dir/local/usercache", + "test-dir/local/usercache/" + user, + "test-dir/local/usercache/" + user + "/appcache", + "test-dir/local/usercache", + "test-dir/local/usercache/" + user, + "test-dir/local/usercache/" + user + "/filecache", + "target/test-dir/local/usercache", + "test-dir/local/usercache/" + user, + "test-dir/local/usercache/" + user + "/appcache", + "test-dir/local/usercache/" + user + "/appcache/" + appId, + "test-dir/logs/" + appId, + "test-dir/local/usercache", + "test-dir/local/usercache/" + user, + "test-dir/local/usercache/" + user + "/appcache", + "test-dir/local/usercache/" + user + "/appcache/" + appId}; + + verify(mockAPI, times(expectedDirs.length)).mkdir(captureMkDir.capture()); + + List capturedPaths = captureMkDir.getAllValues(); + + for (int i = 0; i < expectedDirs.length; ++i) { + assertThat(capturedPaths.get(i).toString(), + CoreMatchers.endsWith(expectedDirs[i])); + } + } + + @Test + public void testStartLocalizerChmod() + throws IOException, InterruptedException { + + String appId = "APP_ID"; + String user = "nobody"; + String locId = "LOC_ID"; + Path nmPrivateContainerTokens = new Path("tokens"); + InetSocketAddress nmAddr = mock(InetSocketAddress.class); + + testedExecutor.startLocalizer(nmPrivateContainerTokens, + nmAddr, user, appId, locId, dirsHandler); + + ArgumentCaptor capturedChmod = + ArgumentCaptor.forClass(java.lang.Short.class); + + verify(mockAPI, times(6)).chmod( + any(Path.class), + capturedChmod.capture()); + + for(Short chmod: capturedChmod.getAllValues()) { + assertEquals((Short) WindowsSecureContainerExecutor.DIR_PERM, chmod); + } + } + + @Test + public void testStartLocalizerChown() + throws IOException, InterruptedException { + + String appId = "APP_ID"; + String user = "nobody"; + String locId = "LOC_ID"; + Path nmPrivateContainerTokens = new Path("tokens"); + InetSocketAddress nmAddr = mock(InetSocketAddress.class); + + testedExecutor.startLocalizer(nmPrivateContainerTokens, + nmAddr, user, appId, locId, dirsHandler); + + ArgumentCaptor captureUser = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captureGroup = ArgumentCaptor.forClass(String.class); + + verify(mockAPI, times(8)).chown(any(Path.class), + captureUser.capture(), captureGroup.capture()); + + for(String capturedUser: captureUser.getAllValues()) { + assertEquals(user, capturedUser); + } + + for(String capturedGroup: captureGroup.getAllValues()) { + assertEquals(yarnGroup, capturedGroup); + } + } + + @Test + public void testLaunchContainerCreateTaskAsUser() + throws IOException, InterruptedException { + + String appId = "APP_ID"; + String user = "nobody"; + String locId = "LOC_ID"; + Path nmPrivateContainerTokens = new Path("tokens"); + Path pidFile = new Path("test-dir/mock_pid_file"); + InetSocketAddress nmAddr = mock(InetSocketAddress.class); + + testedExecutor.activateContainer(mockContainerId, pidFile); + + testedExecutor.launchContainer(mockContainer, + new Path("nm-private-path"), + nmPrivateContainerTokens, user, appId, + new Path("target/test-dir"), + dirsHandler.getLocalDirs(), dirsHandler.getLogDirs()); + + ArgumentCaptor captorCwd = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captorJobName = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captorUserName = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captorPidFile = ArgumentCaptor.forClass(String.class); + ArgumentCaptor captorCmdLine= ArgumentCaptor.forClass(String.class); + + verify(mockAPI).createTaskAsUser( + captorCwd.capture(), captorJobName.capture(), + captorUserName.capture(), captorPidFile.capture(), + captorCmdLine.capture()); + + assertEquals("target\\test-dir", captorCwd.getValue()); + assertEquals(mockContainerId.toString() , captorJobName.getValue()); + assertEquals(user, captorUserName.getValue()); + assertEquals(pidFile.toString(), captorPidFile.getValue()); + assertEquals("cmd /c target/test-dir/default_container_executor.cmd", + captorCmdLine.getValue()); + } +} \ No newline at end of file