diff --git .gitignore .gitignore
index 13b29ff..d8c0c0d 100644
--- .gitignore
+++ .gitignore
@@ -3,6 +3,9 @@
*.iws
*.orig
*.rej
+*.sdf
+*.suo
+*.vcxproj.user
.idea
.svn
.classpath
diff --git hadoop-common-project/hadoop-common/pom.xml hadoop-common-project/hadoop-common/pom.xml
index 09f1c5a..816263a 100644
--- hadoop-common-project/hadoop-common/pom.xml
+++ hadoop-common-project/hadoop-common/pom.xml
@@ -34,6 +34,8 @@
src/test/resources/kdc
common
true
+ ../etc/hadoop
+ wsce-site.xml
@@ -694,6 +696,9 @@
/nologo
/p:Configuration=Release
/p:OutDir=${project.build.directory}/bin/
+ /p:IntermediateOutputPath=${project.build.directory}/winutils/
+ /p:WsceConfigDir=${wsce.config.dir}
+ /p:WsceConfigFile=${wsce.config.file}
diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index a06e3a6..e401fb0 100644
--- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -268,7 +268,11 @@ private FSDataOutputStream create(Path f, boolean overwrite,
throw new IOException("Mkdirs failed to create " + parent.toString());
}
return new FSDataOutputStream(new BufferedOutputStream(
- new LocalFSFileOutputStream(f, false), bufferSize), statistics);
+ createOutputStream(f, false), bufferSize), statistics);
+ }
+
+ protected OutputStream createOutputStream(Path f, boolean append) throws IOException {
+ return new LocalFSFileOutputStream(f, append);
}
@Override
@@ -406,6 +410,10 @@ public boolean delete(Path p, boolean recursive) throws IOException {
}
return Arrays.copyOf(results, j);
}
+
+ protected boolean mkOneDir(File p2f) throws IOException {
+ return p2f.mkdir();
+ }
/**
* Creates the specified directory hierarchy. Does not
@@ -418,8 +426,9 @@ public boolean mkdirs(Path f) throws IOException {
}
Path parent = f.getParent();
File p2f = pathToFile(f);
+ File parent2f = null;
if(parent != null) {
- File parent2f = pathToFile(parent);
+ parent2f = pathToFile(parent);
if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
throw new ParentNotDirectoryException("Parent path is not a directory: "
+ parent);
@@ -429,8 +438,8 @@ public boolean mkdirs(Path f) throws IOException {
throw new FileNotFoundException("Destination exists" +
" and is not a directory: " + p2f.getCanonicalPath());
}
- return (parent == null || mkdirs(parent)) &&
- (p2f.mkdir() || p2f.isDirectory());
+ return (parent == null || parent2f.exists() || mkdirs(parent)) &&
+ (mkOneDir(p2f) || p2f.isDirectory());
}
@Override
diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index fafa295..b897815 100644
--- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -22,6 +22,8 @@
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
@@ -34,6 +36,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.HardLink;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
@@ -502,6 +505,8 @@ public static native void munmap(long addr, long length)
public static final long FILE_BEGIN = 0;
public static final long FILE_CURRENT = 1;
public static final long FILE_END = 2;
+
+ public static final long FILE_ATTRIBUTE_NORMAL = 0x00000080L;
/** Wrapper around CreateFile() on Windows */
public static native FileDescriptor createFile(String path,
@@ -845,4 +850,137 @@ private static native void renameTo0(String src, String dst)
private static native void link0(String src, String dst)
throws NativeIOException;
+
+ public static class Elevated {
+ private static final int MOVE_FILE = 1;
+ private static final int COPY_FILE = 2;
+
+ public static void mkdir(Path dirName) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for mkdir");
+ }
+ elevatedMkDirImpl(dirName.toString());
+ }
+
+ private static native void elevatedMkDirImpl(String dirName) throws IOException;
+
+ public static void chown(Path fileName, String user, String group) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for chown");
+ }
+ elevatedChownImpl(fileName.toString(), user, group);
+ }
+
+ private static native void elevatedChownImpl(String fileName, String user, String group) throws IOException;
+
+ public static void move(Path src, Path dst, boolean replaceExisting) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for move");
+ }
+ elevatedCopyImpl(MOVE_FILE, src.toString(), dst.toString(), replaceExisting);
+ }
+
+ public static void copy(Path src, Path dst, boolean replaceExisting) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for copy");
+ }
+ elevatedCopyImpl(COPY_FILE, src.toString(), dst.toString(), replaceExisting);
+ }
+
+ private static native void elevatedCopyImpl(int operation, String src, String dst, boolean replaceExisting) throws IOException;
+
+ public static void chmod(Path fileName, int mode) {
+
+ }
+
+ public static OutputStream create(Path f, boolean append) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for create");
+ }
+
+ long desiredAccess = Windows.GENERIC_WRITE;
+ long shareMode = 0L;
+ long creationDisposition = append ? Windows.OPEN_ALWAYS : Windows.CREATE_ALWAYS;
+ long flags = Windows.FILE_ATTRIBUTE_NORMAL;
+
+ String fileName = f.toString();
+ fileName = fileName.replace('/', '\\');
+
+ long hFile = elevatedCreateImpl(
+ fileName, desiredAccess, shareMode, creationDisposition, flags);
+ return new FileOutputStream(
+ WinutilsProcessStub.getFileDescriptorFromHandle(hFile));
+ }
+
+ private static native long elevatedCreateImpl(String path, long desiredAccess, long shareMode,
+ long creationDisposition, long flags) throws IOException;
+
+ }
+
+ /**
+ * Wraps a process started by the winutils service helper.
+ *
+ */
+ public static class WinutilsProcessStub extends Process {
+
+ private final long hProcess;
+ private final long hThread;
+ private boolean disposed = false;
+
+ private final InputStream stdErr;
+ private final InputStream stdOut;
+ private final OutputStream stdIn;
+
+ public WinutilsProcessStub(long hProcess, long hThread, long hStdIn, long hStdOut, long hStdErr) {
+ this.hProcess = hProcess;
+ this.hThread = hThread;
+
+ this.stdIn = new FileOutputStream(getFileDescriptorFromHandle(hStdIn));
+ this.stdOut = new FileInputStream(getFileDescriptorFromHandle(hStdOut));
+ this.stdErr = new FileInputStream(getFileDescriptorFromHandle(hStdErr));
+ }
+
+ public static native FileDescriptor getFileDescriptorFromHandle(long handle);
+
+ @Override
+ public native void destroy();
+
+ @Override
+ public native int exitValue();
+
+ @Override
+ public InputStream getErrorStream() {
+ return stdErr;
+ }
+ @Override
+ public InputStream getInputStream() {
+ return stdOut;
+ }
+ @Override
+ public OutputStream getOutputStream() {
+ return stdIn;
+ }
+ @Override
+ public native int waitFor() throws InterruptedException;
+
+ public synchronized native void dispose();
+
+ public native void resume() throws NativeIOException;
+ }
+
+ public synchronized static WinutilsProcessStub createTaskAsUser(
+ String cwd, String jobName, String user, String pidFile, String cmdLine)
+ throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for createTaskAsUser");
+ }
+ synchronized(Shell.WindowsProcessLaunchLock) {
+ return createTaskAsUser0(cwd, jobName, user, pidFile, cmdLine);
+ }
+ }
+
+ private static native WinutilsProcessStub createTaskAsUser0(
+ String cwd, String jobName, String user, String pidFile, String cmdLine)
+ throws NativeIOException;
+
}
diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index fcdc021..67297cd 100644
--- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -643,6 +643,18 @@ public String toString() {
}
}
+ public interface ICommandExecutor {
+
+ void execute() throws IOException;
+
+ int getExitCode() throws IOException;
+
+ String getOutput() throws IOException;
+
+ void dispose();
+
+ }
+
/**
* A simple shell command executor.
*
@@ -651,7 +663,7 @@ public String toString() {
* directory and the environment remains unchanged. The output of the command
* is stored as-is and is expected to be small.
*/
- public static class ShellCommandExecutor extends Shell {
+ public static class ShellCommandExecutor extends Shell implements ICommandExecutor {
private String[] command;
private StringBuffer output;
@@ -743,6 +755,10 @@ public String toString() {
}
return builder.toString();
}
+
+ @Override
+ public void dispose() {
+ }
}
/**
diff --git hadoop-common-project/hadoop-common/src/main/native/native.vcxproj hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
index 0d67e1e..e743788 100644
--- hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
+++ hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
@@ -99,6 +99,7 @@
+
diff --git hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
index d8538c8..ee4db6f 100644
--- hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
+++ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
@@ -49,6 +49,7 @@
#include "file_descriptor.h"
#include "errno_enum.h"
+#include "winutils_process_stub.h"
#define MMAP_PROT_READ org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_READ
#define MMAP_PROT_WRITE org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_WRITE
@@ -68,8 +69,13 @@ static jmethodID nioe_ctor;
// Please see HADOOP-7156 for details.
jobject pw_lock_object;
+/*
+ * Throw a java.IO.IOException, generating the message from errno.
+ * NB. this is also used form winutils_process_stub.c
+ */
+extern void throw_ioe(JNIEnv* env, int errnum);
+
// Internal functions
-static void throw_ioe(JNIEnv* env, int errnum);
#ifdef UNIX
static ssize_t get_pw_buflen();
#endif
@@ -213,7 +219,7 @@ static int map_fadvise_flag(jint flag) {
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_nativeio_NativeIO_initNative(
- JNIEnv *env, jclass clazz) {
+ JNIEnv *env, jclass clazz) {
stat_init(env, clazz);
PASS_EXCEPTIONS_GOTO(env, error);
nioe_init(env);
@@ -224,6 +230,12 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_initNative(
errno_enum_init(env);
PASS_EXCEPTIONS_GOTO(env, error);
#endif
+
+#ifdef WINDOWS
+ winutils_process_stub_init(env);
+ PASS_EXCEPTIONS_GOTO(env, error);
+#endif
+
return;
error:
// these are all idempodent and safe to call even if the
@@ -236,6 +248,9 @@ error:
#ifdef UNIX
errno_enum_deinit(env);
#endif
+#ifdef WINDOWS
+ winutils_process_stub_deinit(env);
+#endif
}
/*
@@ -799,7 +814,7 @@ cleanup:
/*
* Throw a java.IO.IOException, generating the message from errno.
*/
-static void throw_ioe(JNIEnv* env, int errnum)
+void throw_ioe(JNIEnv* env, int errnum)
{
#ifdef UNIX
char message[80];
@@ -1142,6 +1157,279 @@ JNIEnv *env, jclass clazz)
#endif
}
+
+/*
+ * Class: org_apache_hadoop_io_nativeio_NativeIO
+ * Method: createTaskAsUser
+ * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String)Lorg/apache/hadoop/io/nativeio/NativeIO$WinutilsProcessStub
+ */
+JNIEXPORT jobject JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_createTaskAsUser0(JNIEnv* env,
+ jclass clazz, jstring jcwd, jstring jjobName, jstring juser, jstring jpidFile, jstring jcmdLine) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function createTaskAsUser is not supported on Unix");
+ return NULL;
+#endif
+
+#ifdef WINDOWS
+ LPCWSTR cwd = NULL, jobName = NULL,
+ user = NULL, pidFile = NULL, cmdLine = NULL;
+ DWORD dwError = ERROR_SUCCESS;
+ HANDLE hProcess = INVALID_HANDLE_VALUE,
+ hThread = INVALID_HANDLE_VALUE,
+ hStdIn = INVALID_HANDLE_VALUE,
+ hStdOut = INVALID_HANDLE_VALUE,
+ hStdErr = INVALID_HANDLE_VALUE;
+ jobject ret = NULL;
+
+ cwd = (LPCWSTR) (*env)->GetStringChars(env, jcwd, NULL);
+ if (!cwd) goto done; // exception was thrown
+
+ jobName = (LPCWSTR) (*env)->GetStringChars(env, jjobName, NULL);
+ if (!jobName) goto done; // exception was thrown
+
+ user = (LPCWSTR) (*env)->GetStringChars(env, juser, NULL);
+ if (!user) goto done; // exception was thrown
+
+ pidFile = (LPCWSTR) (*env)->GetStringChars(env, jpidFile, NULL);
+ if (!pidFile) goto done; // exception was thrown
+
+ cmdLine = (LPCWSTR) (*env)->GetStringChars(env, jcmdLine, NULL);
+ if (!cmdLine) goto done; // exception was thrown
+
+ LogDebugMessage(L"createTaskAsUser: jcwd:%s job:%s juser:%s pid:%s cmd:%s\n",
+ cwd, jobName, user, pidFile, cmdLine);
+
+ dwError = RpcCall_TaskCreateAsUser(cwd, jobName, user, pidFile, cmdLine,
+ &hProcess, &hThread, &hStdIn, &hStdOut, &hStdErr);
+
+ if (ERROR_SUCCESS == dwError) {
+ ret = winutils_process_stub_create(env, (jlong) hProcess, (jlong) hThread,
+ (jlong) hStdIn, (jlong) hStdOut, (jlong) hStdErr);
+
+ if (NULL == ret) {
+ TerminateProcess(hProcess, EXIT_FAILURE);
+ CloseHandle(hThread);
+ CloseHandle(hProcess);
+ CloseHandle(hStdIn);
+ CloseHandle(hStdOut);
+ CloseHandle(hStdErr);
+ }
+ }
+
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+
+ if (cwd) (*env)->ReleaseStringChars(env, jcwd, cwd);
+ if (jobName) (*env)->ReleaseStringChars(env, jjobName, jobName);
+ if (user) (*env)->ReleaseStringChars(env, juser, user);
+ if (pidFile) (*env)->ReleaseStringChars(env, jpidFile, pidFile);
+ if (cmdLine) (*env)->ReleaseStringChars(env, jcmdLine, cmdLine);
+
+ return ret;
+
+#endif
+}
+
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedChownImpl
+ * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedChownImpl(JNIEnv* env,
+ jclass clazz, jstring jpath, jstring juser, jstring jgroup) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedSetOwner0 is not supported on Unix");
+ return NULL;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR path = NULL, user = NULL, group = NULL;
+ DWORD dwError;
+
+ path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
+ if (!path) goto done; // exception was thrown
+
+ if (juser) {
+ user = (LPCWSTR) (*env)->GetStringChars(env, juser, NULL);
+ if (!user) goto done; // exception was thrown
+ }
+
+ if (jgroup) {
+ group = (LPCWSTR) (*env)->GetStringChars(env, jgroup, NULL);
+ if (!group) goto done; // exception was thrown
+ }
+
+ dwError = RpcCall_WinutilsChown(path, user, group);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (path) (*env)->ReleaseStringChars(env, jpath, path);
+ if (user) (*env)->ReleaseStringChars(env, juser, user);
+ if (group) (*env)->ReleaseStringChars(env, jgroup, group);
+
+#endif
+
+}
+
+
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedMkDirImpl
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedMkDirImpl(JNIEnv* env,
+ jclass clazz, jstring jpath) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedMkDirImpl is not supported on Unix");
+ return NULL;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR path = NULL, user = NULL, group = NULL;
+ DWORD dwError;
+
+ path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
+ if (!path) goto done; // exception was thrown
+
+ dwError = RpcCall_WinutilsMkDir(path);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (path) (*env)->ReleaseStringChars(env, jpath, path);
+
+#endif
+
+}
+
+
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedChmodImpl
+ * Signature: (Ljava/lang/String;I)V
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedChmodImpl(JNIEnv* env,
+ jclass clazz, jstring jpath, jint jmode) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedChmodImpl is not supported on Unix");
+ return NULL;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR path = NULL;
+ DWORD dwError;
+
+ path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
+ if (!path) goto done; // exception was thrown
+
+ dwError = RpcCall_WinutilsChmod(path, (int) jmode);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (path) (*env)->ReleaseStringChars(env, jpath, path);
+
+#endif
+
+}
+
+
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedCopyImpl
+ * Signature: (I;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)V
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedCopyImpl(JNIEnv* env,
+ jclass clazz, jint joperation, jstring jsourcePath, jstring jdestinationPath, jboolean replaceExisting) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedCopyImpl is not supported on Unix");
+ return NULL;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR sourcePath = NULL, destinationPath = NULL;
+ DWORD dwError;
+
+ sourcePath = (LPCWSTR) (*env)->GetStringChars(env, jsourcePath, NULL);
+ if (!sourcePath) goto done; // exception was thrown
+
+ destinationPath = (LPCWSTR) (*env)->GetStringChars(env, jdestinationPath, NULL);
+ if (!destinationPath) goto done; // exception was thrown
+
+ dwError = RpcCall_WinutilsMoveFile((INT) joperation, sourcePath, destinationPath, (BOOL) replaceExisting);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (sourcePath) (*env)->ReleaseStringChars(env, jsourcePath, sourcePath);
+ if (destinationPath) (*env)->ReleaseStringChars(env, jdestinationPath, destinationPath);
+#endif
+}
+
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedCreateImpl
+ * Signature: (Ljava/lang/String;J;J;J;J)J
+ */
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedCreateImpl(JNIEnv* env,
+ jclass clazz, jstring jpath, jlong jdesired_access, jlong jshare_mode, jlong jcreation_disposition, jlong jflags) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedCreateImpl is not supported on Unix");
+ return INVALID_HANDLE_VALUE;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR path = NULL;
+ DWORD dwError;
+ HANDLE hFile = INVALID_HANDLE_VALUE;
+
+ path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
+ if (!path) goto done; // exception was thrown
+
+ dwError = RpcCall_WinutilsCreateFile(path,
+ (DWORD) jdesired_access, (DWORD) jshare_mode, (DWORD) jcreation_disposition, (DWORD) jflags,
+ &hFile);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (path) (*env)->ReleaseStringChars(env, jpath, path);
+ return hFile;
+#endif
+}
+
/**
* vim: sw=2: ts=2: et:
*/
diff --git hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.c hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.c
new file mode 100644
index 0000000..049af45
--- /dev/null
+++ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.c
@@ -0,0 +1,198 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with this
+* work for additional information regarding copyright ownership. The ASF
+* licenses this file to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*/
+
+#include
+#include "org_apache_hadoop.h"
+#include "winutils_process_stub.h"
+#include "winutils.h"
+#include "file_descriptor.h"
+
+// class of org.apache.hadoop.io.nativeio.NativeIO.WinutilsProcessStub
+static jclass wps_class = NULL;
+
+
+static jmethodID wps_constructor = NULL;
+static jfieldID wps_hProcess = NULL;
+static jfieldID wps_hThread = NULL;
+static jfieldID wps_disposed = NULL;
+
+extern void throw_ioe(JNIEnv* env, int errnum);
+
+void winutils_process_stub_init(JNIEnv *env) {
+ if (wps_class != NULL) return; // already initted
+
+ wps_class = (*env)->FindClass(env, WINUTILS_PROCESS_STUB_CLASS);
+ PASS_EXCEPTIONS(env);
+
+ wps_class = (*env)->NewGlobalRef(env, wps_class);
+ PASS_EXCEPTIONS(env);
+
+ wps_hProcess = (*env)->GetFieldID(env, wps_class, "hProcess", "J");
+ PASS_EXCEPTIONS(env);
+
+ wps_hThread = (*env)->GetFieldID(env, wps_class, "hThread", "J");
+ PASS_EXCEPTIONS(env);
+
+ wps_disposed = (*env)->GetFieldID(env, wps_class, "disposed", "Z");
+ PASS_EXCEPTIONS(env);
+
+ wps_constructor = (*env)->GetMethodID(env, wps_class, "", "(JJJJJ)V");
+ PASS_EXCEPTIONS(env);
+
+ LogDebugMessage(L"winutils_process_stub_init\n");
+}
+
+void winutils_process_stub_deinit(JNIEnv *env) {
+ if (wps_class != NULL) {
+ (*env)->DeleteGlobalRef(env, wps_class);
+ wps_class = NULL;
+ }
+ wps_hProcess = NULL;
+ wps_hThread = NULL;
+ wps_disposed = NULL;
+ wps_constructor = NULL;
+ LogDebugMessage(L"winutils_process_stub_deinit\n");
+}
+
+jobject winutils_process_stub_create(JNIEnv *env,
+ jlong hProcess, jlong hThread, jlong hStdIn, jlong hStdOut, jlong hStdErr) {
+ jobject obj = (*env)->NewObject(env, wps_class, wps_constructor,
+ hProcess, hThread, hStdIn, hStdOut, hStdErr);
+ PASS_EXCEPTIONS_RET(env, NULL);
+
+ LogDebugMessage(L"winutils_process_stub_create: %p\n", obj);
+
+ return obj;
+}
+
+
+/*
+ * native void destroy();
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_destroy(
+ JNIEnv *env, jobject objSelf) {
+
+ HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess);
+ LogDebugMessage(L"TerminateProcess: %x\n", hProcess);
+ TerminateProcess(hProcess, EXIT_FAILURE);
+}
+
+/*
+ * native void waitFor();
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_waitFor(
+ JNIEnv *env, jobject objSelf) {
+
+ HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess);
+ LogDebugMessage(L"WaitForSingleObject: %x\n", hProcess);
+ WaitForSingleObject(hProcess, INFINITE);
+}
+
+
+
+/*
+ * native void resume();
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_resume(
+ JNIEnv *env, jobject objSelf) {
+
+ DWORD dwError;
+ HANDLE hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread);
+ if (-1 == ResumeThread(hThread)) {
+ dwError = GetLastError();
+ LogDebugMessage(L"ResumeThread: %x error:%d\n", hThread, dwError);
+ throw_ioe(env, dwError);
+ }
+}
+
+/*
+ * native int exitValue();
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT jint JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_exitValue(
+ JNIEnv *env, jobject objSelf) {
+
+ DWORD exitCode;
+ DWORD dwError;
+ HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess);
+ if (!GetExitCodeProcess(hProcess, &exitCode)) {
+ dwError = GetLastError();
+ throw_ioe(env, dwError);
+ return dwError; // exception was thrown, return value doesn't really matter
+ }
+ LogDebugMessage(L"GetExitCodeProcess: %x :%d\n", hProcess, exitCode);
+
+ return exitCode;
+}
+
+
+/*
+ * native void dispose();
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_dispose(
+ JNIEnv *env, jobject objSelf) {
+
+ HANDLE hProcess = INVALID_HANDLE_VALUE,
+ hThread = INVALID_HANDLE_VALUE;
+
+ jboolean disposed = (*env)->GetBooleanField(env, objSelf, wps_disposed);
+
+ if (JNI_TRUE != disposed) {
+ hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess);
+ hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread);
+
+ CloseHandle(hProcess);
+ CloseHandle(hThread);
+ (*env)->SetBooleanField(env, objSelf, wps_disposed, JNI_TRUE);
+ LogDebugMessage(L"disposed: %p\n", objSelf);
+ }
+}
+
+
+/*
+ * native static FileDescriptor getFileDescriptorFromHandle(long handle);
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT jobject JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_getFileDescriptorFromHandle(
+ JNIEnv *env, jclass klass, jlong handle) {
+
+ LogDebugMessage(L"getFileDescriptorFromHandle: %x\n", handle);
+ return fd_create(env, (long) handle);
+}
+
diff --git hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.h hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.h
new file mode 100644
index 0000000..6ab8ad6
--- /dev/null
+++ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.h
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+
+#define WINUTILS_PROCESS_STUB_CLASS "org/apache/hadoop/io/nativeio/NativeIO$WinutilsProcessStub"
+
+void winutils_process_stub_init(JNIEnv *env);
+void winutils_process_stub_deinit(JNIEnv *env);
+jobject winutils_process_stub_create(JNIEnv *env,
+ jlong hProcess, jlong hThread, jlong hStdIn, jlong hStdOut, jlong hStdErr);
+
+
diff --git hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h
index 92a6b27..3fd5a58 100644
--- hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h
+++ hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h
@@ -32,6 +32,7 @@
#define UNIX
#endif
+
/* A helper macro to 'throw' a java exception. */
#define THROW(env, exception_name, message) \
{ \
diff --git hadoop-common-project/hadoop-common/src/main/winutils/chown.c hadoop-common-project/hadoop-common/src/main/winutils/chown.c
index bc2aefc..d124f73 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/chown.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/chown.c
@@ -18,93 +18,6 @@
#include "winutils.h"
//----------------------------------------------------------------------------
-// Function: ChangeFileOwnerBySid
-//
-// Description:
-// Change a file or directory ownership by giving new owner and group SIDs
-//
-// Returns:
-// ERROR_SUCCESS: on success
-// Error code: otherwise
-//
-// Notes:
-// This function is long path safe, i.e. the path will be converted to long
-// path format if not already converted. So the caller does not need to do
-// the converstion before calling the method.
-//
-static DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
- __in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid)
-{
- LPWSTR longPathName = NULL;
- INT oldMode = 0;
-
- SECURITY_INFORMATION securityInformation = 0;
-
- DWORD dwRtnCode = ERROR_SUCCESS;
-
- // Convert the path the the long path
- //
- dwRtnCode = ConvertToLongPath(path, &longPathName);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- goto ChangeFileOwnerByNameEnd;
- }
-
- // Get a pointer to the existing owner information and DACL
- //
- dwRtnCode = FindFileOwnerAndPermission(longPathName, FALSE, NULL, NULL, &oldMode);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- goto ChangeFileOwnerByNameEnd;
- }
-
- // We need SeTakeOwnershipPrivilege to set the owner if the caller does not
- // have WRITE_OWNER access to the object; we need SeRestorePrivilege if the
- // SID is not contained in the caller's token, and have the SE_GROUP_OWNER
- // permission enabled.
- //
- if (!EnablePrivilege(L"SeTakeOwnershipPrivilege"))
- {
- fwprintf(stdout, L"INFO: The user does not have SeTakeOwnershipPrivilege.\n");
- }
- if (!EnablePrivilege(L"SeRestorePrivilege"))
- {
- fwprintf(stdout, L"INFO: The user does not have SeRestorePrivilege.\n");
- }
-
- assert(pNewOwnerSid != NULL || pNewGroupSid != NULL);
-
- // Set the owners of the file.
- //
- if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION;
- if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION;
- dwRtnCode = SetNamedSecurityInfoW(
- longPathName,
- SE_FILE_OBJECT,
- securityInformation,
- pNewOwnerSid,
- pNewGroupSid,
- NULL,
- NULL);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- goto ChangeFileOwnerByNameEnd;
- }
-
- // Set the permission on the file for the new owner.
- //
- dwRtnCode = ChangeFileModeByMask(longPathName, oldMode);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- goto ChangeFileOwnerByNameEnd;
- }
-
-ChangeFileOwnerByNameEnd:
- LocalFree(longPathName);
- return dwRtnCode;
-}
-
-//----------------------------------------------------------------------------
// Function: Chown
//
// Description:
@@ -130,9 +43,6 @@ int Chown(__in int argc, __in_ecount(argc) wchar_t *argv[])
LPWSTR groupName = NULL;
size_t groupNameLen = 0;
- PSID pNewOwnerSid = NULL;
- PSID pNewGroupSid = NULL;
-
DWORD dwRtnCode = 0;
int ret = EXIT_FAILURE;
@@ -210,48 +120,16 @@ int Chown(__in int argc, __in_ecount(argc) wchar_t *argv[])
goto ChownEnd;
}
- if (userName != NULL)
- {
- dwRtnCode = GetSidFromAcctNameW(userName, &pNewOwnerSid);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- ReportErrorCode(L"GetSidFromAcctName", dwRtnCode);
- fwprintf(stderr, L"Invalid user name: %s\n", userName);
- goto ChownEnd;
- }
- }
-
- if (groupName != NULL)
- {
- dwRtnCode = GetSidFromAcctNameW(groupName, &pNewGroupSid);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- ReportErrorCode(L"GetSidFromAcctName", dwRtnCode);
- fwprintf(stderr, L"Invalid group name: %s\n", groupName);
- goto ChownEnd;
- }
- }
-
- if (wcslen(pathName) == 0 || wcsspn(pathName, L"/?|><:*\"") != 0)
- {
- fwprintf(stderr, L"Incorrect file name format: %s\n", pathName);
- goto ChownEnd;
- }
-
- dwRtnCode = ChangeFileOwnerBySid(pathName, pNewOwnerSid, pNewGroupSid);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- ReportErrorCode(L"ChangeFileOwnerBySid", dwRtnCode);
- goto ChownEnd;
- }
+ dwRtnCode = ChownImpl(userName, groupName, pathName);
+ if (dwRtnCode) {
+ goto ChownEnd;
+ }
ret = EXIT_SUCCESS;
ChownEnd:
LocalFree(userName);
LocalFree(groupName);
- LocalFree(pNewOwnerSid);
- LocalFree(pNewGroupSid);
return ret;
}
diff --git hadoop-common-project/hadoop-common/src/main/winutils/client.c hadoop-common-project/hadoop-common/src/main/winutils/client.c
new file mode 100644
index 0000000..bfe48a3
--- /dev/null
+++ hadoop-common-project/hadoop-common/src/main/winutils/client.c
@@ -0,0 +1,410 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with this
+* work for additional information regarding copyright ownership. The ASF
+* licenses this file to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*/
+
+#include "winutils.h"
+#include
+#include
+#include "hadoopwinutilsvc_h.h"
+
+#pragma comment(lib, "Rpcrt4.lib")
+#pragma comment(lib, "advapi32.lib")
+
+static ACCESS_MASK CLIENT_MASK = 1;
+
+VOID ReportClientError(LPWSTR lpszLocation, DWORD dwError) {
+ LPWSTR debugMsg = NULL;
+ int len;
+ WCHAR hexError[32];
+ HRESULT hr;
+
+ if (IsDebuggerPresent()) {
+ len = FormatMessageW(
+ FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+ NULL, dwError,
+ MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+ (LPWSTR)&debugMsg, 0, NULL);
+
+ LogDebugMessage(L"%s: %s: %x: %.*s\n", GetSystemTimeString(), lpszLocation, dwError, len, debugMsg);
+ }
+
+ if (NULL != debugMsg) LocalFree(debugMsg);
+}
+
+DWORD PrepareRpcBindingHandle(
+ __out RPC_BINDING_HANDLE* pHadoopWinutilsSvcBinding) {
+ DWORD dwError = EXIT_FAILURE;
+ RPC_STATUS status;
+ LPWSTR lpszStringBinding = NULL;
+ ULONG ulCode;
+ RPC_SECURITY_QOS_V3 qos;
+ SID_IDENTIFIER_AUTHORITY authNT = SECURITY_NT_AUTHORITY;
+ BOOL rpcBindingInit = FALSE;
+ PSID pLocalSystemSid = NULL;
+ DWORD cbSystemSidSize = SECURITY_MAX_SID_SIZE;
+
+ pLocalSystemSid = (PSID) LocalAlloc(LPTR, cbSystemSidSize);
+ if (!pLocalSystemSid) {
+ dwError = GetLastError();
+ ReportClientError(L"LocalAlloc", dwError);
+ goto done;
+ }
+
+ if (!CreateWellKnownSid(WinLocalSystemSid, NULL, pLocalSystemSid, &cbSystemSidSize)) {
+ dwError = GetLastError();
+ ReportClientError(L"CreateWellKnownSid", dwError);
+ goto done;
+ }
+
+ ZeroMemory(&qos, sizeof(qos));
+ qos.Version = RPC_C_SECURITY_QOS_VERSION_3;
+ qos.Capabilities = RPC_C_QOS_CAPABILITIES_LOCAL_MA_HINT | RPC_C_QOS_CAPABILITIES_MUTUAL_AUTH;
+ qos.IdentityTracking = RPC_C_QOS_IDENTITY_DYNAMIC;
+ qos.ImpersonationType = RPC_C_IMP_LEVEL_DEFAULT;
+ qos.Sid = pLocalSystemSid;
+
+ status = RpcStringBindingCompose(NULL,
+ SVCBINDING,
+ NULL,
+ SVCNAME,
+ NULL,
+ &lpszStringBinding);
+ if (RPC_S_OK != status) {
+ ReportClientError(L"RpcStringBindingCompose", status);
+ dwError = status;
+ goto done;
+ }
+
+ status = RpcBindingFromStringBinding(lpszStringBinding, pHadoopWinutilsSvcBinding);
+
+ if (RPC_S_OK != status) {
+ ReportClientError(L"RpcBindingFromStringBinding", status);
+ dwError = status;
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ status = RpcBindingSetAuthInfoEx(
+ *pHadoopWinutilsSvcBinding,
+ NULL,
+ RPC_C_AUTHN_LEVEL_PKT_PRIVACY, // AuthnLevel
+ RPC_C_AUTHN_WINNT, // AuthnSvc
+ NULL, // AuthnIdentity (self)
+ RPC_C_AUTHZ_NONE, // AuthzSvc
+ &qos);
+ if (RPC_S_OK != status) {
+ ReportClientError(L"RpcBindingSetAuthInfoEx", status);
+ dwError = status;
+ goto done;
+ }
+
+ dwError = ERROR_SUCCESS;
+
+done:
+
+ if (dwError && rpcBindingInit) RpcBindingFree(pHadoopWinutilsSvcBinding);
+
+ if (pLocalSystemSid) LocalFree(pLocalSystemSid);
+
+ if (NULL != lpszStringBinding) {
+ status = RpcStringFree(&lpszStringBinding);
+ if (RPC_S_OK != status) {
+ ReportClientError(L"RpcStringFree", status);
+ }
+ }
+
+ return dwError;
+}
+
+
+DWORD RpcCall_WinutilsMkDir(
+ __in LPCWSTR filePath) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ MKDIR_REQUEST request;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.filePath = filePath;
+
+ RpcTryExcept {
+ dwError = WinutilsMkDir(hHadoopWinutilsSvcBinding, &request);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ LogDebugMessage(L"RpcCall_WinutilsMkDir: %s :%d\n", filePath, dwError);
+
+ return dwError;
+}
+
+
+
+DWORD RpcCall_WinutilsChown(
+ __in LPCWSTR filePath,
+ __in_opt LPCWSTR ownerName,
+ __in_opt LPCWSTR groupName) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ CHOWN_REQUEST request;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.filePath = filePath;
+ request.ownerName = ownerName;
+ request.groupName = groupName;
+
+ RpcTryExcept {
+ dwError = WinutilsChown(hHadoopWinutilsSvcBinding, &request);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ LogDebugMessage(L"RpcCall_WinutilsChown: %s %s %s :%d\n",
+ ownerName, groupName, filePath, dwError);
+
+ return dwError;
+}
+
+
+DWORD RpcCall_WinutilsChmod(
+ __in LPCWSTR filePath,
+ __in int mode) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ CHMOD_REQUEST request;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.filePath = filePath;
+ request.mode = mode;
+
+ RpcTryExcept {
+ dwError = WinutilsChown(hHadoopWinutilsSvcBinding, &request);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ LogDebugMessage(L"RpcCall_WinutilsChmod: %s %o :%d\n",
+ filePath, mode, dwError);
+
+ return dwError;
+}
+
+
+
+DWORD RpcCall_WinutilsMoveFile(
+ __in int operation,
+ __in LPCWSTR sourcePath,
+ __in LPCWSTR destinationPath,
+ __in BOOL replaceExisting) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ MOVEFILE_REQUEST request;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.operation = operation;
+ request.sourcePath = sourcePath;
+ request.destinationPath = destinationPath;
+ request.replaceExisting = replaceExisting;
+
+ RpcTryExcept {
+ dwError = WinutilsMoveFile(hHadoopWinutilsSvcBinding, &request);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ LogDebugMessage(L"RpcCall_WinutilsMoveFile: %s %s %d :%d\n",
+ sourcePath, destinationPath, replaceExisting, dwError);
+
+ return dwError;
+}
+
+DWORD RpcCall_WinutilsCreateFile(
+ __in LPCWSTR path,
+ __in DWORD desiredAccess,
+ __in DWORD shareMode,
+ __in DWORD creationDisposition,
+ __in DWORD flags,
+ __out HANDLE* hFile) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ DWORD dwSelfPid = GetCurrentProcessId();
+ CREATEFILE_REQUEST request;
+ CREATEFILE_RESPONSE *response = NULL;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.path = path;
+ request.desiredAccess = desiredAccess;
+ request.shareMode = shareMode;
+ request.creationDisposition = creationDisposition;
+ request.flags = flags;
+
+ RpcTryExcept {
+ dwError = WinutilsCreateFile(hHadoopWinutilsSvcBinding, dwSelfPid, &request, &response);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+ if (ERROR_SUCCESS == dwError) {
+ *hFile = response->hFile;
+ }
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ if(NULL != response) MIDL_user_free(response);
+
+ LogDebugMessage(L"RpcCall_WinutilsCreateFile: %s %d, %d, %d, %d :%d\n",
+ path, desiredAccess, shareMode, creationDisposition, flags, dwError);
+
+ return dwError;
+}
+
+DWORD RpcCall_TaskCreateAsUser(
+ LPCWSTR cwd, LPCWSTR jobName,
+ LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine,
+ HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr)
+{
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ DWORD dwSelfPid = GetCurrentProcessId();
+ CREATE_PROCESS_REQUEST request;
+ CREATE_PROCESS_RESPONSE *response = NULL;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.cwd = cwd;
+ request.jobName = jobName;
+ request.user = user;
+ request.pidFile = pidFile;
+ request.cmdLine = cmdLine;
+
+ RpcTryExcept {
+ dwError = WinutilsCreateProcessAsUser(hHadoopWinutilsSvcBinding, dwSelfPid, &request, &response);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+ if (ERROR_SUCCESS == dwError) {
+ *phProcess = response->hProcess;
+ *phThread = response->hThread;
+ *phStdIn = response->hStdIn;
+ *phStdOut = response->hStdOut;
+ *phStdErr = response->hStdErr;
+ }
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ if (NULL != response) {
+ MIDL_user_free(response);
+ }
+
+ return dwError;
+}
+
diff --git hadoop-common-project/hadoop-common/src/main/winutils/config.cpp hadoop-common-project/hadoop-common/src/main/winutils/config.cpp
new file mode 100644
index 0000000..1e07b7f
--- /dev/null
+++ hadoop-common-project/hadoop-common/src/main/winutils/config.cpp
@@ -0,0 +1,174 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with this
+* work for additional information regarding copyright ownership. The ASF
+* licenses this file to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*/
+
+#include "winutils.h"
+#include
+#include
+#import "msxml6.dll"
+
+#define ERROR_CHECK_HRESULT_DONE(hr, message) \
+ if (FAILED(hr)) { \
+ dwError = (DWORD) hr; \
+ LogDebugMessage(L"%s: %x", message, hr); \
+ goto done; \
+ }
+
+DWORD BuildPathRelativeToModule(
+ __in LPCWSTR relativePath,
+ __in size_t len,
+ __out_ecount(len) LPWSTR buffer) {
+ DWORD dwError = ERROR_SUCCESS;
+ WCHAR moduleFile[MAX_PATH];
+ WCHAR modulePath[_MAX_DIR];
+ WCHAR moduleDrive[_MAX_DRIVE];
+ DWORD size;
+ HRESULT hr = S_OK;
+ errno_t errno;
+
+ size = GetModuleFileName(NULL, moduleFile, MAX_PATH);
+ dwError = GetLastError(); // Always check due to ERROR_INSUFFICIENT_BUFFER
+ if (dwError) {
+ LogDebugMessage(L"GetModuleFileName: %x\n", dwError);
+ goto done;
+ }
+
+ errno = _wsplitpath_s(moduleFile,
+ moduleDrive, _MAX_DRIVE,
+ modulePath, _MAX_DIR,
+ NULL, 0, // fname, not interesting
+ NULL, 0); // extenssion, not interesting
+ if (errno) {
+ LogDebugMessage(L"_wsplitpath_s: %x\n", errno);
+ dwError = ERROR_BAD_PATHNAME;
+ goto done;
+ }
+
+ hr = StringCbPrintf(buffer, len, L"%s%s%s", moduleDrive, modulePath, relativePath);
+ if (FAILED(hr)) {
+ // There is no reliable HRESULT to WIN32 mapping, use code.
+ // see http://blogs.msdn.com/b/oldnewthing/archive/2006/11/03/942851.aspx
+ //
+ dwError = HRESULT_CODE(hr);
+ goto done;
+ }
+
+ LogDebugMessage(L"BuildPathRelativeToModule: %s (%s)\n", buffer, relativePath);
+
+done:
+ return dwError;
+}
+
+DWORD GetConfigValue(
+ __in LPCWSTR relativePath,
+ __in LPCWSTR keyName,
+ __out size_t* len, __out_ecount(len) LPCWSTR* value) {
+
+ DWORD dwError = ERROR_SUCCESS;
+ WCHAR xmlPath[MAX_PATH];
+
+ *len = 0;
+ *value = NULL;
+
+ dwError = BuildPathRelativeToModule(
+ relativePath,
+ sizeof(xmlPath)/sizeof(WCHAR),
+ xmlPath);
+
+ if (dwError) {
+ goto done;
+ }
+
+ dwError = GetConfigValueFromXmlFile(xmlPath, keyName, len, value);
+
+done:
+ if (*len) {
+ LogDebugMessage(L"GetConfigValue:%d key:%s len:%d value:%.*s from:%s\n", dwError, keyName, *len, *len, *value, xmlPath);
+ }
+ return dwError;
+}
+
+
+DWORD GetConfigValueFromXmlFile(__in LPCWSTR xmlFile, __in LPCWSTR keyName,
+ __out size_t* outLen, __out_ecount(len) LPCWSTR* outValue) {
+
+ DWORD dwError = ERROR_SUCCESS;
+ HRESULT hr;
+ WCHAR keyXsl[8192];
+ size_t len = 0;
+ LPWSTR value = NULL;
+ BOOL comInitialized = FALSE;
+
+ *outLen = 0;
+ *outValue = NULL;
+
+ hr = CoInitialize(NULL);
+ ERROR_CHECK_HRESULT_DONE(hr, L"CoInitialize");
+ comInitialized = TRUE;
+
+ hr = StringCbPrintf(keyXsl, sizeof(keyXsl), L"//configuration/property[name='%s']/value/text()", keyName);
+ ERROR_CHECK_HRESULT_DONE(hr, L"StringCbPrintf");
+
+ try {
+ MSXML2::IXMLDOMDocument2Ptr pDoc;
+ hr = pDoc.CreateInstance(__uuidof(MSXML2::DOMDocument60), NULL, CLSCTX_INPROC_SERVER);
+ ERROR_CHECK_HRESULT_DONE(hr, L"CreateInstance");
+
+ pDoc->async = VARIANT_FALSE;
+ pDoc->validateOnParse = VARIANT_FALSE;
+ pDoc->resolveExternals = VARIANT_FALSE;
+
+ _variant_t file(xmlFile);
+
+ if (VARIANT_FALSE == pDoc->load(file)) {
+ dwError = pDoc->parseError->errorCode;
+ LogDebugMessage(L"load %s failed:%d %s\n", xmlFile, dwError,
+ static_cast(pDoc->parseError->Getreason()));
+ goto done;
+ }
+
+ MSXML2::IXMLDOMElementPtr pRoot = pDoc->documentElement;
+ MSXML2::IXMLDOMNodePtr keyNode = pRoot->selectSingleNode(keyXsl);
+
+ if (keyNode) {
+ _bstr_t bstrValue = static_cast<_bstr_t>(keyNode->nodeValue);
+ len = bstrValue.length();
+ value = (LPWSTR) LocalAlloc(LPTR, (len+1) * sizeof(WCHAR));
+ LPCWSTR lpwszValue = static_cast(bstrValue);
+ memcpy(value, lpwszValue, (len) * sizeof(WCHAR));
+ LogDebugMessage(L"key:%s :%.*s [%s]\n", keyName, len, value, lpwszValue);
+ *outLen = len;
+ *outValue = value;
+ }
+ else {
+ LogDebugMessage(L"node Xpath:%s not found in:%s\n", keyXsl, xmlFile);
+ }
+ }
+ catch(_com_error errorObject) {
+ dwError = errorObject.Error();
+ LogDebugMessage(L"catch _com_error:%x %s\n", dwError, errorObject.ErrorMessage());
+ goto done;
+ }
+
+done:
+ if (comInitialized) {
+ CoUninitialize();
+ }
+
+ return dwError;
+}
+
+
diff --git hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl
new file mode 100644
index 0000000..ec7128f
--- /dev/null
+++ hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import "oaidl.idl";
+import "ocidl.idl";
+
+[
+ uuid(0492311C-1718-4F53-A6EB-86AD7039988D),
+ version(1.0),
+ pointer_default(unique),
+ //implicit_handle(handle_t hHadoopWinutilsSvcBinding),
+ endpoint("ncalrpc:[hadoopwinutilsvc]"),
+#ifndef __midl
+ explicit_handle
+#endif
+]
+interface HadoopWinutilSvc
+{
+ typedef struct {
+ [string] const wchar_t* cwd;
+ [string] const wchar_t* jobName;
+ [string] const wchar_t* user;
+ [string] const wchar_t* pidFile;
+ [string] const wchar_t* cmdLine;
+ } CREATE_PROCESS_REQUEST;
+
+ typedef struct {
+ LONG_PTR hProcess;
+ LONG_PTR hThread;
+ LONG_PTR hStdIn;
+ LONG_PTR hStdOut;
+ LONG_PTR hStdErr;
+ } CREATE_PROCESS_RESPONSE;
+
+ typedef struct {
+ [string] const wchar_t* filePath;
+ [string] const wchar_t* ownerName;
+ [string] const wchar_t* groupName;
+ } CHOWN_REQUEST;
+
+ typedef struct {
+ [string] const wchar_t* filePath;
+ int mode;
+ } CHMOD_REQUEST;
+
+ typedef struct {
+ [string] const wchar_t* filePath;
+ } MKDIR_REQUEST;
+
+ typedef enum { MOVE_FILE = 1, COPY_FILE = 2} MOVE_COPY_OPERATION;
+
+ typedef struct {
+ MOVE_COPY_OPERATION operation;
+ [string] const wchar_t* sourcePath;
+ [string] const wchar_t* destinationPath;
+ boolean replaceExisting;
+ } MOVEFILE_REQUEST;
+
+ typedef struct {
+ [string] const wchar_t* path;
+ int desiredAccess;
+ int shareMode;
+ int creationDisposition;
+ int flags;
+ } CREATEFILE_REQUEST;
+
+ typedef struct {
+ LONG_PTR hFile;
+ } CREATEFILE_RESPONSE;
+
+ error_status_t WinutilsMkDir(
+ [in] MKDIR_REQUEST *request);
+
+ error_status_t WinutilsMoveFile(
+ [in] MOVEFILE_REQUEST *request);
+
+ error_status_t WinutilsChown(
+ [in] CHOWN_REQUEST *request);
+
+ error_status_t WinutilsChmod(
+ [in] CHMOD_REQUEST *request);
+
+ error_status_t WinutilsCreateFile(
+ [in] int nmPid,
+ [in] CREATEFILE_REQUEST *request,
+ [out] CREATEFILE_RESPONSE **response);
+
+ error_status_t WinutilsCreateProcessAsUser(
+ [in] int nmPid,
+ [in] CREATE_PROCESS_REQUEST *request,
+ [out] CREATE_PROCESS_RESPONSE **response);
+
+}
diff --git hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
index 1c0007a..7be86c3 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
+++ hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
@@ -27,6 +27,13 @@
#include
#include
#include
+#include
+#include
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
enum EXIT_CODE
{
@@ -36,6 +43,12 @@ enum EXIT_CODE
FAILURE = EXIT_FAILURE,
/* Failure code indicates the user does not privilege to create symlinks */
SYMLINK_NO_PRIVILEGE = 2,
+
+ ERROR_TASK_NOT_ALIVE = 1,
+
+ // This exit code for killed processes is compatible with Unix, where a killed
+ // process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137.
+ KILLED_PROCESS_EXIT_CODE = 137,
};
@@ -153,6 +166,99 @@ DWORD ChangeFileModeByMask(__in LPCWSTR path, INT mode);
DWORD GetLocalGroupsForUser(__in LPCWSTR user,
__out LPLOCALGROUP_USERS_INFO_0 *groups, __out LPDWORD entries);
-BOOL EnablePrivilege(__in LPCWSTR privilegeName);
-
void GetLibraryName(__in LPCVOID lpAddress, __out LPWSTR *filename);
+
+DWORD EnablePrivilege(__in LPCWSTR privilegeName);
+
+void AssignLsaString(__inout LSA_STRING * target, __in const char *strBuf);
+
+DWORD RegisterWithLsa(__in const char *logonProcessName, __out HANDLE * lsaHandle);
+
+void UnregisterWithLsa(__in HANDLE lsaHandle);
+
+DWORD LookupKerberosAuthenticationPackageId(__in HANDLE lsaHandle, __out ULONG * packageId);
+
+DWORD CreateLogonForUser(__in HANDLE lsaHandle,
+ __in const char * tokenSourceName,
+ __in const char * tokenOriginName,
+ __in ULONG authnPkgId,
+ __in const wchar_t* principalName,
+ __out HANDLE *tokenHandle);
+
+DWORD LoadUserProfileForLogon(__in HANDLE logonHandle, __out PROFILEINFO * pi);
+
+DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi);
+
+DWORD RunService(__in int argc, __in_ecount(argc) wchar_t *argv[]);
+void ServiceUsage();
+
+
+DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
+ __in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid);
+
+DWORD ChownImpl(
+ __in_opt LPCWSTR userName,
+ __in_opt LPCWSTR groupName,
+ __in LPCWSTR pathName);
+
+LPCWSTR GetSystemTimeString();
+
+VOID LogDebugMessage(LPCWSTR format, ...);
+
+DWORD SplitStringIgnoreSpaceW(
+ __in size_t len,
+ __in_ecount(len) LPCWSTR source,
+ __in WCHAR deli,
+ __out size_t* count, __out_ecount(count) WCHAR*** out);
+
+DWORD GetConfigValue(
+ __in LPCWSTR relativePath,
+ __in LPCWSTR keyName,
+ __out size_t* len,
+ __out_ecount(len) LPCWSTR* value);
+DWORD GetConfigValueFromXmlFile(
+ __in LPCWSTR xmlFile,
+ __in LPCWSTR keyName,
+ __out size_t* len,
+ __out_ecount(len) LPCWSTR* value);
+
+
+DWORD BuildServiceSecurityDescriptor(
+ __in ACCESS_MASK accessMask,
+ __in size_t grantSidCount,
+ __in_ecount(grantSidCount) PSID* pGrantSids,
+ __in size_t denySidCount,
+ __in_ecount(denySidCount) PSID* pDenySids,
+ __out PSECURITY_DESCRIPTOR* pSD);
+
+extern const WCHAR* wsceConfigRelativePath;
+
+extern LPCWSTR NM_WSCE_ALLOWED;
+
+
+#define SVCNAME TEXT("hadoopwinutilsvc")
+#define SVCBINDING TEXT("ncalrpc")
+
+DWORD RpcCall_TaskCreateAsUser(
+ LPCWSTR cwd, LPCWSTR jobName,
+ LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine,
+ HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr);
+
+DWORD RpcCall_WinutilsCreateFile(
+ __in LPCWSTR path,
+ __in DWORD desiredAccess,
+ __in DWORD shareMode,
+ __in DWORD creationDisposition,
+ __in DWORD flags,
+ __out HANDLE* hFile);
+
+DWORD RpcCall_WinutilsMoveFile(
+ __in LPCWSTR sourcePath,
+ __in LPCWSTR destinationPath,
+ __in BOOL replaceExisting);
+
+#ifdef __cplusplus
+}
+#endif
+
+
diff --git hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
index 391247f..78a3de3 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
@@ -17,9 +17,26 @@
#pragma comment(lib, "authz.lib")
#pragma comment(lib, "netapi32.lib")
+#pragma comment(lib, "Secur32.lib")
+#pragma comment(lib, "Userenv.lib")
+#pragma comment(lib, "Ntdsapi.lib")
+
#include "winutils.h"
+#include
+#include
#include
#include
+#include
+#include
+
+#define WIDEN_STRING(x) WIDEN_STRING_(x)
+#define WIDEN_STRING_(x) L ## x
+#define STRINGIFY(x) STRINGIFY_(x)
+#define STRINGIFY_(x) #x
+
+
+#pragma message("WSCE config is " STRINGIFY(WSCE_CONFIG_DIR) "\\" STRINGIFY(WSCE_CONFIG_FILE))
+const WCHAR* wsceConfigRelativePath = WIDEN_STRING(STRINGIFY(WSCE_CONFIG_DIR)) L"\\" WIDEN_STRING(STRINGIFY(WSCE_CONFIG_FILE));
/*
* The array of 12 months' three-letter abbreviations
@@ -235,10 +252,10 @@ ConvertToLongPathExit:
// Function: IsDirFileInfo
//
// Description:
-// Test if the given file information is a directory
+// Test if the given file information is a directory
//
// Returns:
-// TRUE if it is a directory
+// TRUE if it is a directory
// FALSE otherwise
//
// Notes:
@@ -255,10 +272,10 @@ BOOL IsDirFileInfo(const BY_HANDLE_FILE_INFORMATION *fileInformation)
// Function: CheckFileAttributes
//
// Description:
-// Check if the given file has all the given attribute(s)
+// Check if the given file has all the given attribute(s)
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -279,10 +296,10 @@ static DWORD FileAttributesCheck(
// Function: IsDirectory
//
// Description:
-// Check if the given file is a directory
+// Check if the given file is a directory
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -296,10 +313,10 @@ DWORD DirectoryCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: IsReparsePoint
//
// Description:
-// Check if the given file is a reparse point
+// Check if the given file is a reparse point
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -313,10 +330,10 @@ static DWORD ReparsePointCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: CheckReparseTag
//
// Description:
-// Check if the given file is a reparse point of the given tag.
+// Check if the given file is a reparse point of the given tag.
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -354,10 +371,10 @@ static DWORD ReparseTagCheck(__in LPCWSTR path, __in DWORD tag, __out PBOOL res)
// Function: IsSymbolicLink
//
// Description:
-// Check if the given file is a symbolic link.
+// Check if the given file is a symbolic link.
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -371,10 +388,10 @@ DWORD SymbolicLinkCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: IsJunctionPoint
//
// Description:
-// Check if the given file is a junction point.
+// Check if the given file is a junction point.
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -388,14 +405,14 @@ DWORD JunctionPointCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: GetSidFromAcctNameW
//
// Description:
-// To retrieve the SID for a user account
+// To retrieve the SID for a user account
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
// Other error code: otherwise
//
// Notes:
-// Caller needs to destroy the memory of Sid by calling LocalFree()
+// Caller needs to destroy the memory of Sid by calling LocalFree()
//
DWORD GetSidFromAcctNameW(__in PCWSTR acctName, __out PSID *ppSid)
{
@@ -477,10 +494,10 @@ DWORD GetSidFromAcctNameW(__in PCWSTR acctName, __out PSID *ppSid)
// Function: GetUnixAccessMask
//
// Description:
-// Compute the 3 bit Unix mask for the owner, group, or, others
+// Compute the 3 bit Unix mask for the owner, group, or, others
//
// Returns:
-// The 3 bit Unix mask in INT
+// The 3 bit Unix mask in INT
//
// Notes:
//
@@ -504,10 +521,10 @@ static INT GetUnixAccessMask(ACCESS_MASK Mask)
// Function: GetAccess
//
// Description:
-// Get Windows acces mask by AuthZ methods
+// Get Windows acces mask by AuthZ methods
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
//
// Notes:
//
@@ -552,10 +569,10 @@ static DWORD GetAccess(AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzClient,
// Function: GetEffectiveRightsForSid
//
// Description:
-// Get Windows acces mask by AuthZ methods
+// Get Windows acces mask by AuthZ methods
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
//
// Notes:
// We run into problems for local user accounts when using the method
@@ -712,11 +729,11 @@ CheckAccessEnd:
// Function: FindFileOwnerAndPermissionByHandle
//
// Description:
-// Find the owner, primary group and permissions of a file object given the
+// Find the owner, primary group and permissions of a file object given the
// the file object handle. The function will always follow symbolic links.
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
// Error code otherwise
//
// Notes:
@@ -776,10 +793,10 @@ FindFileOwnerAndPermissionByHandleEnd:
// Function: FindFileOwnerAndPermission
//
// Description:
-// Find the owner, primary group and permissions of a file object
+// Find the owner, primary group and permissions of a file object
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
// Error code otherwise
//
// Notes:
@@ -797,7 +814,6 @@ DWORD FindFileOwnerAndPermission(
__out_opt PINT pMask)
{
DWORD dwRtnCode = 0;
-
PSECURITY_DESCRIPTOR pSd = NULL;
PSID psidOwner = NULL;
@@ -1206,14 +1222,14 @@ static DWORD GetWindowsDACLs(__in INT unixMask,
if (winUserAccessDenyMask &&
!AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winUserAccessDenyMask, pOwnerSid))
{
ret = GetLastError();
goto GetWindowsDACLsEnd;
}
if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winUserAccessAllowMask, pOwnerSid))
{
ret = GetLastError();
@@ -1221,21 +1237,21 @@ static DWORD GetWindowsDACLs(__in INT unixMask,
}
if (winGroupAccessDenyMask &&
!AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winGroupAccessDenyMask, pGroupSid))
{
ret = GetLastError();
goto GetWindowsDACLsEnd;
}
if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winGroupAccessAllowMask, pGroupSid))
{
ret = GetLastError();
goto GetWindowsDACLsEnd;
}
if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winOtherAccessAllowMask, pEveryoneSid))
{
ret = GetLastError();
@@ -1439,14 +1455,14 @@ ChangeFileModeByMaskEnd:
// Function: GetAccntNameFromSid
//
// Description:
-// To retrieve an account name given the SID
+// To retrieve an account name given the SID
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
// Other error code: otherwise
//
// Notes:
-// Caller needs to destroy the memory of account name by calling LocalFree()
+// Caller needs to destroy the memory of account name by calling LocalFree()
//
DWORD GetAccntNameFromSid(__in PSID pSid, __out PWSTR *ppAcctName)
{
@@ -1535,10 +1551,10 @@ GetAccntNameFromSidEnd:
// Function: GetLocalGroupsForUser
//
// Description:
-// Get an array of groups for the given user.
+// Get an array of groups for the given user.
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// Other error code on failure
//
// Notes:
@@ -1630,19 +1646,21 @@ GetLocalGroupsForUserEnd:
return ret;
}
+
//----------------------------------------------------------------------------
// Function: EnablePrivilege
//
// Description:
-// Check if the process has the given privilege. If yes, enable the privilege
+// Check if the process has the given privilege. If yes, enable the privilege
// to the process's access token.
//
// Returns:
-// TRUE: on success
+// ERROR_SUCCESS on success
+// GetLastError() on error
//
// Notes:
//
-BOOL EnablePrivilege(__in LPCWSTR privilegeName)
+DWORD EnablePrivilege(__in LPCWSTR privilegeName)
{
HANDLE hToken = INVALID_HANDLE_VALUE;
TOKEN_PRIVILEGES tp = { 0 };
@@ -1651,28 +1669,31 @@ BOOL EnablePrivilege(__in LPCWSTR privilegeName)
if (!OpenProcessToken(GetCurrentProcess(),
TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken))
{
- ReportErrorCode(L"OpenProcessToken", GetLastError());
- return FALSE;
+ dwErrCode = GetLastError();
+ ReportErrorCode(L"OpenProcessToken", dwErrCode);
+ return dwErrCode;
}
tp.PrivilegeCount = 1;
if (!LookupPrivilegeValueW(NULL,
privilegeName, &(tp.Privileges[0].Luid)))
{
- ReportErrorCode(L"LookupPrivilegeValue", GetLastError());
+ dwErrCode = GetLastError();
+ ReportErrorCode(L"LookupPrivilegeValue", dwErrCode);
CloseHandle(hToken);
- return FALSE;
+ return dwErrCode;
}
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
// As stated on MSDN, we need to use GetLastError() to check if
// AdjustTokenPrivileges() adjusted all of the specified privileges.
//
- AdjustTokenPrivileges(hToken, FALSE, &tp, 0, NULL, NULL);
+ if( !AdjustTokenPrivileges(hToken, FALSE, &tp, 0, NULL, NULL) ) {
dwErrCode = GetLastError();
+ }
CloseHandle(hToken);
- return dwErrCode == ERROR_SUCCESS;
+ return dwErrCode;
}
//----------------------------------------------------------------------------
@@ -1701,12 +1722,15 @@ void ReportErrorCode(LPCWSTR func, DWORD err)
(LPWSTR)&msg, 0, NULL);
if (len > 0)
{
+ LogDebugMessage(L"%s error (%d): %s\n", func, err, msg);
fwprintf(stderr, L"%s error (%d): %s\n", func, err, msg);
}
else
{
+ LogDebugMessage(L"%s error code: %d.\n", func, err);
fwprintf(stderr, L"%s error code: %d.\n", func, err);
}
+
if (msg != NULL) LocalFree(msg);
}
@@ -1716,9 +1740,6 @@ void ReportErrorCode(LPCWSTR func, DWORD err)
// Description:
// Given an address, get the file name of the library from which it was loaded.
//
-// Returns:
-// None
-//
// Notes:
// - The function allocates heap memory and points the filename out parameter to
// the newly allocated memory, which will contain the name of the file.
@@ -1757,3 +1778,820 @@ cleanup:
*filename = NULL;
}
}
+
+// Function: AssignLsaString
+//
+// Description:
+// fills in values of LSA_STRING struct to point to a string buffer
+//
+// Returns:
+// None
+//
+// IMPORTANT*** strBuf is not copied. It must be globally immutable
+//
+void AssignLsaString(__inout LSA_STRING * target, __in const char *strBuf)
+{
+ target->Length = (USHORT)(sizeof(char)*strlen(strBuf));
+ target->MaximumLength = target->Length;
+ target->Buffer = (char *)(strBuf);
+}
+
+//----------------------------------------------------------------------------
+// Function: RegisterWithLsa
+//
+// Description:
+// Registers with local security authority and sets handle for use in later LSA
+// operations
+//
+// Returns:
+// ERROR_SUCCESS on success
+// Other error code on failure
+//
+// Notes:
+//
+DWORD RegisterWithLsa(__in const char *logonProcessName, __out HANDLE * lsaHandle)
+{
+ LSA_STRING processName;
+ LSA_OPERATIONAL_MODE o_mode; // never useful as per msdn docs
+ NTSTATUS registerStatus;
+ *lsaHandle = 0;
+
+ AssignLsaString(&processName, logonProcessName);
+ registerStatus = LsaRegisterLogonProcess(&processName, lsaHandle, &o_mode);
+
+ return LsaNtStatusToWinError( registerStatus );
+}
+
+//----------------------------------------------------------------------------
+// Function: UnregisterWithLsa
+//
+// Description:
+// Closes LSA handle allocated by RegisterWithLsa()
+//
+// Returns:
+// None
+//
+// Notes:
+//
+void UnregisterWithLsa(__in HANDLE lsaHandle)
+{
+ LsaClose(lsaHandle);
+}
+
+//----------------------------------------------------------------------------
+// Function: LookupKerberosAuthenticationPackageId
+//
+// Description:
+// Looks of the current id (integer index) of the Kerberos authentication package on the local
+// machine.
+//
+// Returns:
+// ERROR_SUCCESS on success
+// Other error code on failure
+//
+// Notes:
+//
+DWORD LookupKerberosAuthenticationPackageId(__in HANDLE lsaHandle, __out ULONG * packageId)
+{
+ NTSTATUS lookupStatus;
+ LSA_STRING pkgName;
+
+ AssignLsaString(&pkgName, MICROSOFT_KERBEROS_NAME_A);
+ lookupStatus = LsaLookupAuthenticationPackage(lsaHandle, &pkgName, packageId);
+ return LsaNtStatusToWinError( lookupStatus );
+}
+
+//----------------------------------------------------------------------------
+// Function: CreateLogonForUser
+//
+// Description:
+// Contacts the local LSA and performs a logon without credential for the
+// given principal. This logon token will be local machine only and have no
+// network credentials attached.
+//
+// Returns:
+// ERROR_SUCCESS on success
+// Other error code on failure
+//
+// Notes:
+// This call assumes that all required privileges have already been enabled (TCB etc).
+// IMPORTANT **** tokenOriginName must be immutable!
+//
+DWORD CreateLogonForUser(__in HANDLE lsaHandle,
+ __in const char * tokenSourceName,
+ __in const char * tokenOriginName, // must be immutable, will not be copied!
+ __in ULONG authnPkgId,
+ __in const wchar_t* principalName,
+ __out HANDLE *tokenHandle)
+{
+ DWORD logonStatus = ERROR_ASSERTION_FAILURE; // Failure to set status should trigger error
+ TOKEN_SOURCE tokenSource;
+ LSA_STRING originName;
+ void * profile = NULL;
+
+ // from MSDN:
+ // The ClientUpn and ClientRealm members of the KERB_S4U_LOGON
+ // structure must point to buffers in memory that are contiguous
+ // to the structure itself. The value of the
+ // AuthenticationInformationLength parameter must take into
+ // account the length of these buffers.
+ const int principalNameBufLen = lstrlen(principalName)*sizeof(*principalName);
+ const int totalAuthInfoLen = sizeof(KERB_S4U_LOGON) + principalNameBufLen;
+ KERB_S4U_LOGON* s4uLogonAuthInfo = (KERB_S4U_LOGON*)calloc(totalAuthInfoLen, 1);
+ if (s4uLogonAuthInfo == NULL ) {
+ logonStatus = ERROR_NOT_ENOUGH_MEMORY;
+ goto done;
+ }
+ s4uLogonAuthInfo->MessageType = KerbS4ULogon;
+ s4uLogonAuthInfo->ClientUpn.Buffer = (wchar_t*)((char*)s4uLogonAuthInfo + sizeof *s4uLogonAuthInfo);
+ CopyMemory(s4uLogonAuthInfo->ClientUpn.Buffer, principalName, principalNameBufLen);
+ s4uLogonAuthInfo->ClientUpn.Length = (USHORT)principalNameBufLen;
+ s4uLogonAuthInfo->ClientUpn.MaximumLength = (USHORT)principalNameBufLen;
+
+ AllocateLocallyUniqueId(&tokenSource.SourceIdentifier);
+ StringCchCopyA(tokenSource.SourceName, TOKEN_SOURCE_LENGTH, tokenSourceName );
+ AssignLsaString(&originName, tokenOriginName);
+
+ {
+ DWORD cbProfile = 0;
+ LUID logonId;
+ QUOTA_LIMITS quotaLimits;
+ NTSTATUS subStatus;
+
+ NTSTATUS logonNtStatus = LsaLogonUser(lsaHandle,
+ &originName,
+ Batch, // SECURITY_LOGON_TYPE
+ authnPkgId,
+ s4uLogonAuthInfo,
+ totalAuthInfoLen,
+ 0,
+ &tokenSource,
+ &profile,
+ &cbProfile,
+ &logonId,
+ tokenHandle,
+ "aLimits,
+ &subStatus);
+ logonStatus = LsaNtStatusToWinError( logonNtStatus );
+ }
+done:
+ // clean up
+ if (s4uLogonAuthInfo != NULL) {
+ free(s4uLogonAuthInfo);
+ }
+ if (profile != NULL) {
+ LsaFreeReturnBuffer(profile);
+ }
+ return logonStatus;
+}
+
+// NOTE: must free allocatedName
+DWORD GetNameFromLogonToken(__in HANDLE logonToken, __out wchar_t **allocatedName)
+{
+ DWORD userInfoSize = 0;
+ PTOKEN_USER user = NULL;
+ DWORD userNameSize = 0;
+ wchar_t * userName = NULL;
+ DWORD domainNameSize = 0;
+ wchar_t * domainName = NULL;
+ SID_NAME_USE sidUse = SidTypeUnknown;
+ DWORD getNameStatus = ERROR_ASSERTION_FAILURE; // Failure to set status should trigger error
+ BOOL tokenInformation = FALSE;
+
+ // call for sid size then alloc and call for sid
+ tokenInformation = GetTokenInformation(logonToken, TokenUser, NULL, 0, &userInfoSize);
+ assert (FALSE == tokenInformation);
+
+ // last call should have failed and filled in allocation size
+ if ((getNameStatus = GetLastError()) != ERROR_INSUFFICIENT_BUFFER)
+ {
+ goto done;
+ }
+ user = (PTOKEN_USER)calloc(userInfoSize,1);
+ if (user == NULL)
+ {
+ getNameStatus = ERROR_NOT_ENOUGH_MEMORY;
+ goto done;
+ }
+ if (!GetTokenInformation(logonToken, TokenUser, user, userInfoSize, &userInfoSize)) {
+ getNameStatus = GetLastError();
+ goto done;
+ }
+ LookupAccountSid( NULL, user->User.Sid, NULL, &userNameSize, NULL, &domainNameSize, &sidUse );
+ // last call should have failed and filled in allocation size
+ if ((getNameStatus = GetLastError()) != ERROR_INSUFFICIENT_BUFFER)
+ {
+ goto done;
+ }
+ userName = (wchar_t *)calloc(userNameSize, sizeof(wchar_t));
+ if (userName == NULL) {
+ getNameStatus = ERROR_NOT_ENOUGH_MEMORY;
+ goto done;
+ }
+ domainName = (wchar_t *)calloc(domainNameSize, sizeof(wchar_t));
+ if (domainName == NULL) {
+ getNameStatus = ERROR_NOT_ENOUGH_MEMORY;
+ goto done;
+ }
+ if (!LookupAccountSid( NULL, user->User.Sid, userName, &userNameSize, domainName, &domainNameSize, &sidUse )) {
+ getNameStatus = GetLastError();
+ goto done;
+ }
+
+ getNameStatus = ERROR_SUCCESS;
+ *allocatedName = userName;
+ userName = NULL;
+done:
+ if (user != NULL) {
+ free( user );
+ user = NULL;
+ }
+ if (userName != NULL) {
+ free( userName );
+ userName = NULL;
+ }
+ if (domainName != NULL) {
+ free( domainName );
+ domainName = NULL;
+ }
+ return getNameStatus;
+}
+
+DWORD LoadUserProfileForLogon(__in HANDLE logonHandle, __out PROFILEINFO * pi)
+{
+ wchar_t *userName = NULL;
+ DWORD loadProfileStatus = ERROR_ASSERTION_FAILURE; // Failure to set status should trigger error
+
+ loadProfileStatus = GetNameFromLogonToken( logonHandle, &userName );
+ if (loadProfileStatus != ERROR_SUCCESS) {
+ goto done;
+ }
+
+ assert(pi);
+
+ ZeroMemory( pi, sizeof(*pi) );
+ pi->dwSize = sizeof(*pi);
+ pi->lpUserName = userName;
+ pi->dwFlags = PI_NOUI;
+
+ // if the profile does not exist it will be created
+ if ( !LoadUserProfile( logonHandle, pi ) ) {
+ loadProfileStatus = GetLastError();
+ goto done;
+ }
+
+ loadProfileStatus = ERROR_SUCCESS;
+done:
+ return loadProfileStatus;
+}
+
+
+
+DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi)
+{
+ DWORD touchProfileStatus = ERROR_ASSERTION_FAILURE; // Failure to set status should trigger error
+
+ assert(pi);
+
+ if ( !UnloadUserProfile(logonHandle, pi->hProfile ) ) {
+ touchProfileStatus = GetLastError();
+ goto done;
+ }
+ if (pi->lpUserName != NULL) {
+ free(pi->lpUserName);
+ pi->lpUserName = NULL;
+ }
+ ZeroMemory( pi, sizeof(*pi) );
+
+ touchProfileStatus = ERROR_SUCCESS;
+done:
+ return touchProfileStatus;
+}
+
+
+//----------------------------------------------------------------------------
+// Function: ChangeFileOwnerBySid
+//
+// Description:
+// Change a file or directory ownership by giving new owner and group SIDs
+//
+// Returns:
+// ERROR_SUCCESS: on success
+// Error code: otherwise
+//
+// Notes:
+// This function is long path safe, i.e. the path will be converted to long
+// path format if not already converted. So the caller does not need to do
+// the converstion before calling the method.
+//
+DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
+ __in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid)
+{
+ LPWSTR longPathName = NULL;
+ INT oldMode = 0;
+
+ SECURITY_INFORMATION securityInformation = 0;
+
+ DWORD dwRtnCode = ERROR_SUCCESS;
+
+ // Convert the path the the long path
+ //
+ dwRtnCode = ConvertToLongPath(path, &longPathName);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ goto ChangeFileOwnerByNameEnd;
+ }
+
+ // Get a pointer to the existing owner information and DACL
+ //
+ dwRtnCode = FindFileOwnerAndPermission(longPathName, FALSE, NULL, NULL, &oldMode);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ goto ChangeFileOwnerByNameEnd;
+ }
+
+ // We need SeTakeOwnershipPrivilege to set the owner if the caller does not
+ // have WRITE_OWNER access to the object; we need SeRestorePrivilege if the
+ // SID is not contained in the caller's token, and have the SE_GROUP_OWNER
+ // permission enabled.
+ //
+ if (EnablePrivilege(L"SeTakeOwnershipPrivilege") != ERROR_SUCCESS)
+ {
+ fwprintf(stdout, L"INFO: The user does not have SeTakeOwnershipPrivilege.\n");
+ }
+ if (EnablePrivilege(L"SeRestorePrivilege") != ERROR_SUCCESS)
+ {
+ fwprintf(stdout, L"INFO: The user does not have SeRestorePrivilege.\n");
+ }
+
+ assert(pNewOwnerSid != NULL || pNewGroupSid != NULL);
+
+ // Set the owners of the file.
+ //
+ if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION;
+ if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION;
+ dwRtnCode = SetNamedSecurityInfoW(
+ longPathName,
+ SE_FILE_OBJECT,
+ securityInformation,
+ pNewOwnerSid,
+ pNewGroupSid,
+ NULL,
+ NULL);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ goto ChangeFileOwnerByNameEnd;
+ }
+
+ // Set the permission on the file for the new owner.
+ //
+ dwRtnCode = ChangeFileModeByMask(longPathName, oldMode);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ goto ChangeFileOwnerByNameEnd;
+ }
+
+ChangeFileOwnerByNameEnd:
+ LocalFree(longPathName);
+ return dwRtnCode;
+}
+
+
+
+DWORD ChownImpl(
+ __in_opt LPCWSTR userName,
+ __in_opt LPCWSTR groupName,
+ __in LPCWSTR pathName) {
+
+ DWORD dwError;
+
+ PSID pNewOwnerSid = NULL;
+ PSID pNewGroupSid = NULL;
+
+ if (userName != NULL)
+ {
+ dwError = GetSidFromAcctNameW(userName, &pNewOwnerSid);
+ if (dwError != ERROR_SUCCESS)
+ {
+ ReportErrorCode(L"GetSidFromAcctName", dwError);
+ fwprintf(stderr, L"Invalid user name: %s\n", userName);
+ goto done;
+ }
+ }
+
+ if (groupName != NULL)
+ {
+ dwError = GetSidFromAcctNameW(groupName, &pNewGroupSid);
+ if (dwError != ERROR_SUCCESS)
+ {
+ ReportErrorCode(L"GetSidFromAcctName", dwError);
+ fwprintf(stderr, L"Invalid group name: %s\n", groupName);
+ goto done;
+ }
+ }
+
+ if (wcslen(pathName) == 0 || wcsspn(pathName, L"/?|><:*\"") != 0)
+ {
+ fwprintf(stderr, L"Incorrect file name format: %s\n", pathName);
+ goto done;
+ }
+
+ dwError = ChangeFileOwnerBySid(pathName, pNewOwnerSid, pNewGroupSid);
+ if (dwError != ERROR_SUCCESS)
+ {
+ ReportErrorCode(L"ChangeFileOwnerBySid", dwError);
+ goto done;
+ }
+done:
+ LocalFree(pNewOwnerSid);
+ LocalFree(pNewGroupSid);
+
+ return dwError;
+}
+
+
+
+LPCWSTR GetSystemTimeString() {
+ __declspec(thread) static WCHAR buffer[1024];
+ DWORD dwError;
+ FILETIME ftime;
+ SYSTEMTIME systime;
+ LARGE_INTEGER counter, frequency;
+ int subSec;
+ double qpc;
+ HRESULT hr;
+ buffer[0] = L'\0';
+
+ // GetSystemTimePreciseAsFileTime is only available in Win8+ and our libs do not link against it
+
+ GetSystemTimeAsFileTime(&ftime);
+
+ if (!FileTimeToSystemTime(&ftime, &systime)) {
+ dwError = GetLastError();
+ LogDebugMessage(L"FileTimeToSystemTime error:%d\n", dwError);
+ goto done;
+ }
+
+ // Get the ms from QPC. GetSystemTimeAdjustment is ignored...
+
+ QueryPerformanceCounter(&counter);
+ QueryPerformanceFrequency(&frequency);
+
+ qpc = (double) counter.QuadPart / (double) frequency.QuadPart;
+ subSec = ((qpc - (long)qpc) * 1000000);
+
+ hr = StringCbPrintf(buffer, sizeof(buffer), L"%02d:%02d:%02d.%06d",
+ (int)systime.wHour, (int)systime.wMinute, (int)systime.wSecond, (int)subSec);
+
+ if (FAILED(hr)) {
+ LogDebugMessage(L"StringCbPrintf error:%d\n", hr);
+ }
+done:
+ return buffer;
+}
+
+
+//----------------------------------------------------------------------------
+// Function: LogDebugMessage
+//
+// Description:
+// Sends a message to the debugger console, if one is attached
+//
+// Notes:
+// Native debugger: windbg, ntsd, cdb, visual studio
+//
+VOID LogDebugMessage(LPCWSTR format, ...) {
+ LPWSTR buffer[8192];
+ va_list args;
+ HRESULT hr;
+
+ if (!IsDebuggerPresent()) return;
+
+ va_start(args, format);
+ hr = StringCbVPrintf(buffer, sizeof(buffer), format, args);
+ if (SUCCEEDED(hr)) {
+ OutputDebugString(buffer);
+ }
+ va_end(args);
+}
+
+//----------------------------------------------------------------------------
+// Function: SplitStringIgnoreSpaceW
+//
+// Description:
+// splits a null-terminated string based on a delimiter
+//
+// Returns:
+// ERROR_SUCCESS: on success
+// error code: otherwise
+//
+// Notes:
+// The tokes are also null-terminated
+// Caller should use LocalFree to clear outTokens
+//
+DWORD SplitStringIgnoreSpaceW(
+ __in size_t len,
+ __in_ecount(len) LPCWSTR source,
+ __in WCHAR deli,
+ __out size_t* count,
+ __out_ecount(count) WCHAR*** outTokens) {
+
+ size_t tokenCount = 0;
+ size_t crtSource;
+ size_t crtToken = 0;
+ WCHAR* lpwszTokenStart = NULL;
+ WCHAR* lpwszTokenEnd = NULL;
+ WCHAR* lpwszBuffer = NULL;
+ size_t tokenLength = 0;
+ size_t cchBufferLength = 0;
+ WCHAR crt;
+ WCHAR** tokens = NULL;
+ enum {BLANK, TOKEN, DELIMITER} State = BLANK;
+
+ for(crtSource = 0; crtSource < len; ++crtSource) {
+ crt = source[crtSource];
+ switch(State) {
+ case BLANK: // intentional fallthrough
+ case DELIMITER:
+ if (crt == deli) {
+ State = DELIMITER;
+ }
+ else if (!iswspace(crt)) {
+ ++tokenCount;
+ lpwszTokenEnd = lpwszTokenStart = source + crtSource;
+ State = TOKEN;
+ }
+ else {
+ State = BLANK;
+ }
+ break;
+ case TOKEN:
+ if (crt == deli) {
+ State = DELIMITER;
+ cchBufferLength += lpwszTokenEnd - lpwszTokenStart + 2;
+ }
+ else if (!iswspace(crt)) {
+ lpwszTokenEnd = source + crtSource;
+ }
+ break;
+ }
+ }
+
+ if (State == TOKEN) {
+ cchBufferLength += lpwszTokenEnd - lpwszTokenStart + 2;
+ }
+
+ LogDebugMessage(L"counted %d [buffer:%d] tokens in %s\n", tokenCount, cchBufferLength, source);
+
+ #define COPY_CURRENT_TOKEN \
+ tokenLength = lpwszTokenEnd - lpwszTokenStart + 1; \
+ tokens[crtToken] = lpwszBuffer; \
+ memcpy(tokens[crtToken], lpwszTokenStart, tokenLength*sizeof(WCHAR)); \
+ tokens[crtToken][tokenLength] = L'\0'; \
+ lpwszBuffer += (tokenLength+1); \
+ ++crtToken;
+
+ if (tokenCount) {
+
+ // We use one contigous memory for both the pointer arrays and the data copy buffers
+ // We cannot use in-place references (zero-copy) because the function users
+ // need null-terminated strings for the tokens
+
+ tokens = (WCHAR**) LocalAlloc(LPTR,
+ sizeof(WCHAR*) * tokenCount + // for the pointers
+ sizeof(WCHAR) * cchBufferLength); // for the data
+
+ // Data will be copied after the array
+ lpwszBuffer = (WCHAR*)(((BYTE*)tokens) + (sizeof(WCHAR*) * tokenCount));
+
+ State = BLANK;
+
+ for(crtSource = 0; crtSource < len; ++crtSource) {
+ crt = source[crtSource];
+ switch(State) {
+ case DELIMITER: // intentional fallthrough
+ case BLANK:
+ if (crt == deli) {
+ State = DELIMITER;
+ }
+ else if (!iswspace(crt)) {
+ lpwszTokenEnd = lpwszTokenStart = source + crtSource;
+ State = TOKEN;
+ }
+ else {
+ State = BLANK;
+ }
+ break;
+ case TOKEN:
+ if (crt == deli) {
+ COPY_CURRENT_TOKEN;
+ State = DELIMITER;
+ }
+ else if (!iswspace(crt)) {
+ lpwszTokenEnd = source + crtSource;
+ }
+ break;
+ }
+ }
+
+ // Copy out last token, if any
+ if (TOKEN == State) {
+ COPY_CURRENT_TOKEN;
+ }
+ }
+
+ *count = tokenCount;
+ *outTokens = tokens;
+
+ return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: BuildServiceSecurityDescriptor
+//
+// Description:
+// Builds a security descriptor for an arbitrary object
+//
+// Returns:
+// ERROR_SUCCESS: on success
+// error code: otherwise
+//
+// Notes:
+// The SD is a of the self-contained flavor (offsets, not pointers)
+// Caller should use LocalFree to clear allocated pSD
+//
+DWORD BuildServiceSecurityDescriptor(
+ __in ACCESS_MASK accessMask,
+ __in size_t grantSidCount,
+ __in_ecount(grantSidCount) PSID* pGrantSids,
+ __in size_t denySidCount,
+ __in_ecount(denySidCount) PSID* pDenySids,
+ __out PSECURITY_DESCRIPTOR* pSD) {
+
+ DWORD dwError = ERROR_SUCCESS;
+ int crt = 0;
+ int len = 0;
+ EXPLICIT_ACCESS* eas = NULL;
+ LPWSTR lpszSD = NULL;
+ ULONG cchSD = 0;
+ HANDLE hToken = INVALID_HANDLE_VALUE;
+ DWORD dwBufferSize = 0;
+ PTOKEN_USER pTokenUser = NULL;
+ PTOKEN_PRIMARY_GROUP pTokenGroup = NULL;
+ PSECURITY_DESCRIPTOR pTempSD = NULL;
+ ULONG cbSD = 0;
+ TRUSTEE owner, group;
+
+ ZeroMemory(&owner, sizeof(owner));
+
+ // We'll need our own SID to add as SD owner
+ if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &hToken)) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ if (!GetTokenInformation(hToken, TokenUser, NULL, 0, &dwBufferSize)) {
+ dwError = GetLastError();
+ if (ERROR_INSUFFICIENT_BUFFER != dwError) {
+ goto done;
+ }
+ }
+
+ pTokenUser = (PTOKEN_USER) LocalAlloc(LPTR, dwBufferSize);
+ if (NULL == pTokenUser) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ if (!GetTokenInformation(hToken, TokenUser, pTokenUser, dwBufferSize, &dwBufferSize)) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ if (!IsValidSid(pTokenUser->User.Sid)) {
+ dwError = ERROR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ dwBufferSize = 0;
+ if (!GetTokenInformation(hToken, TokenPrimaryGroup, NULL, 0, &dwBufferSize)) {
+ dwError = GetLastError();
+ if (ERROR_INSUFFICIENT_BUFFER != dwError) {
+ goto done;
+ }
+ }
+
+ pTokenGroup = (PTOKEN_USER) LocalAlloc(LPTR, dwBufferSize);
+ if (NULL == pTokenUser) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ if (!GetTokenInformation(hToken, TokenPrimaryGroup, pTokenGroup, dwBufferSize, &dwBufferSize)) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ if (!IsValidSid(pTokenGroup->PrimaryGroup)) {
+ dwError = ERROR_INVALID_PARAMETER;
+ goto done;
+ }
+
+ owner.TrusteeForm = TRUSTEE_IS_SID;
+ owner.TrusteeType = TRUSTEE_IS_UNKNOWN;
+ owner.ptstrName = (LPCWSTR) pTokenUser->User.Sid;
+
+ group.TrusteeForm = TRUSTEE_IS_SID;
+ group.TrusteeType = TRUSTEE_IS_UNKNOWN;
+ group.ptstrName = (LPCWSTR) pTokenGroup->PrimaryGroup;
+
+ eas = (EXPLICIT_ACCESS*) LocalAlloc(LPTR, sizeof(EXPLICIT_ACCESS) * (grantSidCount + denySidCount));
+
+ // Build the granted list
+ for (crt = 0; crt < grantSidCount; ++crt) {
+ eas[crt].grfAccessPermissions = accessMask;
+ eas[crt].grfAccessMode = GRANT_ACCESS;
+ eas[crt].grfInheritance = NO_INHERITANCE;
+ eas[crt].Trustee.TrusteeForm = TRUSTEE_IS_SID;
+ eas[crt].Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN;
+ eas[crt].Trustee.ptstrName = (LPCWSTR) pGrantSids[crt];
+ eas[crt].Trustee.pMultipleTrustee = NULL;
+ eas[crt].Trustee.MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE;
+ }
+
+ // Build the deny list
+ for (; crt < grantSidCount + denySidCount; ++crt) {
+ eas[crt].grfAccessPermissions = accessMask;
+ eas[crt].grfAccessMode = DENY_ACCESS;
+ eas[crt].grfInheritance = NO_INHERITANCE;
+ eas[crt].Trustee.TrusteeForm = TRUSTEE_IS_SID;
+ eas[crt].Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN;
+ eas[crt].Trustee.ptstrName = (LPCWSTR) pDenySids[crt - grantSidCount];
+ eas[crt].Trustee.pMultipleTrustee = NULL;
+ eas[crt].Trustee.MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE;
+ }
+
+ dwError = BuildSecurityDescriptor(
+ &owner,
+ &group,
+ crt,
+ eas,
+ 0, // cCountOfAuditEntries
+ NULL, // pListOfAuditEntries
+ NULL, // pOldSD
+ &cbSD,
+ &pTempSD);
+ if (ERROR_SUCCESS != dwError) {
+ goto done;
+ }
+
+ *pSD = pTempSD;
+ pTempSD = NULL;
+
+ if (IsDebuggerPresent()) {
+ ConvertSecurityDescriptorToStringSecurityDescriptor(*pSD,
+ SDDL_REVISION_1,
+ DACL_SECURITY_INFORMATION,
+ &lpszSD,
+ &cchSD);
+ LogDebugMessage(L"pSD: %.*s\n", cchSD, lpszSD);
+ }
+
+done:
+ if (eas) LocalFree(eas);
+ if (pTokenUser) LocalFree(pTokenUser);
+ if (INVALID_HANDLE_VALUE != hToken) CloseHandle(hToken);
+ if (lpszSD) LocalFree(lpszSD);
+ if (pTempSD) LocalFree(pTempSD);
+ return dwError;
+}
+
+
+//----------------------------------------------------------------------------
+// Function: MIDL_user_allocate
+//
+// Description:
+// Hard-coded function name used by RPC midl code for allocations
+//
+// Notes:
+// Must match the de-allocation mechanism used in MIDL_user_free
+//
+void __RPC_FAR * __RPC_USER MIDL_user_allocate(size_t len)
+{
+ return LocalAlloc(LPTR, len);
+}
+
+ //----------------------------------------------------------------------------
+ // Function: MIDL_user_free
+ //
+ // Description:
+ // Hard-coded function name used by RPC midl code for deallocations
+ //
+ // NoteS:
+ // Must match the allocation mechanism used in MIDL_user_allocate
+ //
+void __RPC_USER MIDL_user_free(void __RPC_FAR * ptr)
+{
+ LocalFree(ptr);
+}
+
diff --git hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
index fc0519d..37b7f31 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
+++ hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
@@ -19,18 +19,10 @@
-
- Debug
- Win32
-
Debug
x64
-
- Release
- Win32
-
Release
x64
@@ -42,22 +34,11 @@
winutils
-
- StaticLibrary
- true
- Unicode
-
StaticLibrary
true
Unicode
-
- StaticLibrary
- false
- true
- Unicode
-
StaticLibrary
false
@@ -67,15 +48,9 @@
-
-
-
-
-
-
@@ -83,74 +58,35 @@
include;$(IncludePath)
-
- true
-
true
-
- ..\..\..\target\winutils\$(Configuration)\
-
-
- false
false
- ..\..\..\target\bin\
- ..\..\..\target\winutils\$(Platform)\$(Configuration)\
-
-
-
-
- Level3
- Disabled
- WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
-
-
- Console
- true
-
-
Level4
Disabled
- WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ WIN32;_DEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)
Console
true
-
-
- Level3
-
-
- MaxSpeed
- true
- true
- WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
-
-
- Console
- true
- true
- true
-
-
Level3
- MaxSpeed
+
+ Disabled
true
true
- WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ WIN32;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)
Console
@@ -159,12 +95,34 @@
true
+
+
+ $(IntermediateOutputPath)
+
+
+ true
+ X64
+ $(IntermediateOutputPath)
+ true
+ true
+ true
+ 2
+
+
+
+
+
+
+
+
+
+
diff --git hadoop-common-project/hadoop-common/src/main/winutils/main.c hadoop-common-project/hadoop-common/src/main/winutils/main.c
index 0f40774..ac73aec 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/main.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/main.c
@@ -17,12 +17,27 @@
#include "winutils.h"
+#include
+
static void Usage(LPCWSTR program);
+LONG WINAPI WinutilsSehUnhandled(_In_ struct _EXCEPTION_POINTERS *ecxr) {
+ LogDebugMessage(L"unhandled SEH: code:%x flags:%d\n",
+ ecxr->ExceptionRecord->ExceptionCode,
+ ecxr->ExceptionRecord->ExceptionFlags);
+ fwprintf(stderr, L"Unhandled exception code:%x at address:%p",
+ ecxr->ExceptionRecord->ExceptionCode,
+ ecxr->ExceptionRecord->ExceptionAddress);
+ ExitProcess(ERROR_UNHANDLED_EXCEPTION);
+ return EXCEPTION_EXECUTE_HANDLER; // not that it matters...
+}
+
int wmain(__in int argc, __in_ecount(argc) wchar_t* argv[])
{
LPCWSTR cmd = NULL;
+ SetUnhandledExceptionFilter(WinutilsSehUnhandled);
+
if (argc < 2)
{
Usage(argv[0]);
@@ -67,6 +82,10 @@ int wmain(__in int argc, __in_ecount(argc) wchar_t* argv[])
{
return SystemInfo();
}
+ else if (wcscmp(L"service", cmd) == 0)
+ {
+ return RunService(argc - 1, argv + 1);
+ }
else if (wcscmp(L"help", cmd) == 0)
{
Usage(argv[0]);
@@ -119,5 +138,9 @@ The available commands and their usages are:\n\n", program);
fwprintf(stdout, L"%-15s%s\n\n", L"task", L"Task operations.");
TaskUsage();
+
+ fwprintf(stdout, L"%-15s%s\n\n", L"service", L"Service operations.");
+ ServiceUsage();
+
fwprintf(stdout, L"\n\n");
}
diff --git hadoop-common-project/hadoop-common/src/main/winutils/service.c hadoop-common-project/hadoop-common/src/main/winutils/service.c
new file mode 100644
index 0000000..37b8628
--- /dev/null
+++ hadoop-common-project/hadoop-common/src/main/winutils/service.c
@@ -0,0 +1,1033 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with this
+* work for additional information regarding copyright ownership. The ASF
+* licenses this file to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*/
+
+#include "winutils.h"
+#include "winutils_msg.h"
+#include
+#include
+#include
+#include
+#include
+#include
+#include "hadoopwinutilsvc_h.h"
+
+#pragma comment(lib, "Rpcrt4.lib")
+#pragma comment(lib, "advapi32.lib")
+#pragma comment(lib, "authz.lib")
+
+LPCWSTR NM_WSCE_ALLOWED = L"yarn.nodemanager.windows-secure-container-executor.allowed";
+
+#define SERVICE_ACCESS_MASK 0x00000001
+
+SERVICE_STATUS gSvcStatus;
+SERVICE_STATUS_HANDLE gSvcStatusHandle;
+HANDLE ghSvcStopEvent = INVALID_HANDLE_VALUE;
+HANDLE ghWaitObject = INVALID_HANDLE_VALUE;
+HANDLE ghEventLog = INVALID_HANDLE_VALUE;
+BOOL isListenning = FALSE;
+PSECURITY_DESCRIPTOR pAllowedSD = NULL;
+
+VOID SvcError(DWORD dwError);
+VOID WINAPI SvcMain(DWORD dwArg, LPTSTR* lpszArgv);
+DWORD SvcInit();
+DWORD RpcInit();
+DWORD AuthInit();
+VOID ReportSvcStatus( DWORD dwCurrentState,
+ DWORD dwWin32ExitCode,
+ DWORD dwWaitHint);
+VOID WINAPI SvcCtrlHandler( DWORD dwCtrl );
+VOID CALLBACK SvcShutdown(
+ _In_ PVOID lpParameter,
+ _In_ BOOLEAN TimerOrWaitFired);
+
+#define CHECK_ERROR_DONE(status, expected, category, message) \
+ if (status != expected) { \
+ ReportSvcCheckError( \
+ EVENTLOG_ERROR_TYPE, \
+ category, \
+ status, \
+ message); \
+ goto done; \
+ } else { \
+ LogDebugMessage(L"%s: OK\n", message); \
+ }
+
+
+#define CHECK_RPC_STATUS_DONE(status, message) \
+ CHECK_ERROR_DONE(status, RPC_S_OK, SERVICE_CATEGORY, message)
+
+#define CHECK_SVC_STATUS_DONE(status, message) \
+ CHECK_ERROR_DONE(status, ERROR_SUCCESS, SERVICE_CATEGORY, message)
+
+#define CHECK_UNWIND_RPC(rpcCall) { \
+ unwindStatus = rpcCall; \
+ if (RPC_S_OK != unwindStatus) { \
+ ReportSvcCheckError( \
+ EVENTLOG_WARNING_TYPE, \
+ SERVICE_CATEGORY, \
+ unwindStatus, \
+ L#rpcCall); \
+ } \
+ }
+
+
+//----------------------------------------------------------------------------
+// Function: ReportSvcCheckError
+//
+// Description:
+// Reports an error with the system event log and to debugger console (if present)
+//
+void ReportSvcCheckError(WORD type, WORD category, DWORD dwError, LPCWSTR message) {
+ int len;
+ LPWSTR systemMsg = NULL;
+ LPWSTR appMsg = NULL;
+ DWORD dwReportError;
+ LPWSTR reportMsg = NULL;
+ WCHAR hexError[32];
+ LPCWSTR inserts[] = {message, NULL, NULL, NULL};
+ HRESULT hr;
+
+ hr = StringCbPrintf(hexError, sizeof(hexError), TEXT("%x"), dwError);
+ if (SUCCEEDED(hr)) {
+ inserts[1] = hexError;
+ }
+ else {
+ inserts[1] = L"(Failed to format dwError as string)";
+ }
+
+ len = FormatMessageW(
+ FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+ NULL, dwError,
+ MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+ (LPWSTR)&systemMsg, 0, NULL);
+
+ if (len) {
+ inserts[2] = systemMsg;
+ }
+ else {
+ inserts[2] = L"(Failed to get the system error message)";
+ }
+
+ LogDebugMessage(L"%s:%d %.*s\n", message, dwError, len, systemMsg);
+
+ if (INVALID_HANDLE_VALUE != ghEventLog) {
+ if (!ReportEvent(ghEventLog, type, category, MSG_CHECK_ERROR,
+ NULL, // lpUserSid
+ (WORD) 3, // wNumStrings
+ (DWORD) 0, // dwDataSize
+ inserts, // *lpStrings
+ NULL // lpRawData
+ )) {
+ // We tried to report and failed. Send to dbg.
+ dwReportError = GetLastError();
+ len = FormatMessageW(
+ FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+ NULL, dwReportError,
+ MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+ (LPWSTR)&reportMsg, 0, NULL);
+ LogDebugMessage(L"ReportEvent: Error:%d %.*s\n", dwReportError, reportMsg);
+ }
+ };
+
+ if (NULL != systemMsg) LocalFree(systemMsg);
+ if (NULL != reportMsg) LocalFree(reportMsg);
+}
+
+
+VOID ReportSvcMessage(WORD type, WORD category, DWORD msgId) {
+ DWORD dwError;
+
+ if (INVALID_HANDLE_VALUE != ghEventLog) {
+ if (!ReportEvent(ghEventLog, type, category, msgId,
+ NULL, // lpUserSid
+ (WORD) 0, // wNumStrings
+ (DWORD) 0, // dwDataSize
+ NULL, // *lpStrings
+ NULL // lpRawData
+ )) {
+ // We tried to report and failed but debugger is attached. Send to dbg.
+ dwError = GetLastError();
+ LogDebugMessage(L"ReportEvent: error %d\n", dwError);
+ }
+ }
+}
+
+
+//----------------------------------------------------------------------------
+// Function: RunService
+//
+// Description:
+// Registers with NT SCM and starts the service
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// Error code otherwise: otherwise
+DWORD RunService(__in int argc, __in_ecount(argc) wchar_t *argv[])
+{
+ DWORD dwError= ERROR_SUCCESS;
+ int argStart = 1;
+
+ static const SERVICE_TABLE_ENTRY serviceTable[] = {
+ { SVCNAME, (LPSERVICE_MAIN_FUNCTION) SvcMain },
+ { NULL, NULL }
+ };
+
+ dwError = AuthInit();
+ if (ERROR_SUCCESS != dwError) {
+ SvcError(dwError);
+ goto done;
+ }
+
+ ghEventLog = RegisterEventSource(NULL, SVCNAME);
+ if (NULL == ghEventLog) {
+ dwError = GetLastError();
+ CHECK_SVC_STATUS_DONE(dwError, L"RegisterEventSource")
+ }
+
+ if (!StartServiceCtrlDispatcher(serviceTable)) {
+ dwError = GetLastError();
+ CHECK_SVC_STATUS_DONE(dwError, L"StartServiceCtrlDispatcher")
+ }
+
+done:
+ return dwError;
+}
+
+//----------------------------------------------------------------------------
+// Function: SvcMain
+//
+// Description:
+// Service main entry point.
+//
+VOID WINAPI SvcMain() {
+ DWORD dwError = ERROR_SUCCESS;
+
+ gSvcStatusHandle = RegisterServiceCtrlHandler(
+ SVCNAME,
+ SvcCtrlHandler);
+ if( !gSvcStatusHandle ) {
+ dwError = GetLastError();
+ CHECK_SVC_STATUS_DONE(dwError, L"RegisterServiceCtrlHandler")
+ }
+
+ // These SERVICE_STATUS members remain as set here
+ gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
+ gSvcStatus.dwServiceSpecificExitCode = 0;
+
+ // Report initial status to the SCM
+ ReportSvcStatus( SERVICE_START_PENDING, NO_ERROR, 3000 );
+
+ // Perform service-specific initialization and work.
+ dwError = SvcInit();
+
+done:
+ return;
+}
+
+//----------------------------------------------------------------------------
+// Function: SvcInit
+//
+// Description:
+// Initializes the service.
+//
+DWORD SvcInit() {
+ DWORD dwError = ERROR_SUCCESS;
+
+ dwError = EnablePrivilege(SE_DEBUG_NAME);
+ if( dwError != ERROR_SUCCESS ) {
+ goto done;
+ }
+
+ // The recommended way to shutdown the service is to use an event
+ // and attach a callback with RegisterWaitForSingleObject
+ //
+ ghSvcStopEvent = CreateEvent(
+ NULL, // default security attributes
+ TRUE, // manual reset event
+ FALSE, // not signaled
+ NULL); // no name
+
+ if ( ghSvcStopEvent == NULL)
+ {
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"CreateEvent");
+ ReportSvcStatus( SERVICE_STOPPED, dwError, 0 );
+ goto done;
+ }
+
+ if (!RegisterWaitForSingleObject (&ghWaitObject,
+ ghSvcStopEvent,
+ SvcShutdown,
+ NULL,
+ INFINITE,
+ WT_EXECUTEONLYONCE)) {
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"RegisterWaitForSingleObject");
+ CloseHandle(ghSvcStopEvent);
+ ReportSvcStatus( SERVICE_STOPPED, dwError, 0 );
+ goto done;
+ }
+
+ // Report running status when initialization is complete.
+ ReportSvcStatus( SERVICE_RUNNING, NO_ERROR, 0 );
+
+ dwError = RpcInit();
+
+done:
+ return dwError;
+}
+
+//----------------------------------------------------------------------------
+// Function: RpcAuthorizeCallback
+//
+// Description:
+// RPC Authorization callback.
+//
+// Returns:
+// RPC_S_OK for access authorized
+// RPC_S_ACCESS_DENIED for access denied
+//
+RPC_STATUS CALLBACK RpcAuthorizeCallback (
+ RPC_IF_HANDLE hInterface,
+ void* pContext)
+{
+ RPC_STATUS status,
+ unwindStatus,
+ authStatus = RPC_S_ACCESS_DENIED;
+ DWORD dwError;
+ LUID luidReserved2;
+ AUTHZ_ACCESS_REQUEST request;
+ AUTHZ_ACCESS_REPLY reply;
+ AUTHZ_CLIENT_CONTEXT_HANDLE hClientContext = NULL;
+ DWORD authError = ERROR_SUCCESS;
+ DWORD saclResult = 0;
+ ACCESS_MASK grantedMask = 0;
+
+ ZeroMemory(&luidReserved2, sizeof(luidReserved2));
+ ZeroMemory(&request, sizeof(request));
+ ZeroMemory(&reply, sizeof(reply));
+
+ status = RpcGetAuthorizationContextForClient(NULL,
+ FALSE, // ImpersonateOnReturn
+ NULL, // Reserved1
+ NULL, // pExpirationTime
+ luidReserved2, // Reserved2
+ 0, // Reserved3
+ NULL, // Reserved4
+ &hClientContext);
+ CHECK_RPC_STATUS_DONE(status, L"RpcGetAuthorizationContextForClient");
+
+ request.DesiredAccess = MAXIMUM_ALLOWED;
+ reply.Error = &authError;
+ reply.SaclEvaluationResults = &saclResult;
+ reply.ResultListLength = 1;
+ reply.GrantedAccessMask = &grantedMask;
+
+ if (!AuthzAccessCheck(
+ 0,
+ hClientContext,
+ &request,
+ NULL, // AuditEvent
+ pAllowedSD,
+ NULL, // OptionalSecurityDescriptorArray
+ 0, // OptionalSecurityDescriptorCount
+ &reply,
+ NULL // phAccessCheckResults
+ )) {
+ dwError = GetLastError();
+ CHECK_SVC_STATUS_DONE(dwError, L"AuthzAccessCheck");
+ }
+
+ LogDebugMessage(L"AutzAccessCheck: Error:%d sacl:%d access:%d\n",
+ authError, saclResult, grantedMask);
+ if (authError == ERROR_SUCCESS && (grantedMask & SERVICE_ACCESS_MASK)) {
+ authStatus = RPC_S_OK;
+ }
+
+done:
+ if (NULL != hClientContext) CHECK_UNWIND_RPC(RpcFreeAuthorizationContext(&hClientContext));
+ return authStatus;
+}
+
+//----------------------------------------------------------------------------
+// Function: AuthInit
+//
+// Description:
+// Initializes the authorization structures (security descriptor).
+//
+// Notes:
+// This is called from RunService solely for debugging purposed
+// so that it can be tested by wimply running winutil service from CLI (no SCM)
+//
+DWORD AuthInit() {
+ DWORD dwError = ERROR_SUCCESS;
+ int count = 0;
+ int crt = 0;
+ int len = 0;
+ LPCWSTR value = NULL;
+ WCHAR** tokens = NULL;
+ LPWSTR lpszSD = NULL;
+ ULONG cchSD = 0;
+ DWORD dwBufferSize = 0;
+ int allowedCount = 0;
+ PSID* allowedSids = NULL;
+
+
+ dwError = GetConfigValue(
+ wsceConfigRelativePath,
+ NM_WSCE_ALLOWED, &len, &value);
+ CHECK_SVC_STATUS_DONE(dwError, L"GetConfigValue");
+
+ if (0 == len) {
+ CHECK_SVC_STATUS_DONE(ERROR_BAD_CONFIGURATION, NM_WSCE_ALLOWED);
+ }
+
+ dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens);
+ CHECK_SVC_STATUS_DONE(dwError, L"SplitStringIgnoreSpaceW");
+
+ allowedSids = (PSID*) LocalAlloc(LPTR, sizeof(PSID) * count);
+ for (crt = 0; crt < count; ++crt) {
+ dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]);
+ CHECK_SVC_STATUS_DONE(dwError, L"GetSidFromAcctNameW");
+ }
+
+ allowedCount = count;
+
+ dwError = BuildServiceSecurityDescriptor(SERVICE_ACCESS_MASK,
+ allowedCount, allowedSids, 0, NULL, &pAllowedSD);
+ CHECK_SVC_STATUS_DONE(dwError, L"BuildServiceSecurityDescriptor");
+
+done:
+ if (lpszSD) LocalFree(lpszSD);
+ if (value) LocalFree(value);
+ if (tokens) LocalFree(tokens);
+ return dwError;
+}
+
+//----------------------------------------------------------------------------
+// Function: RpcInit
+//
+// Description:
+// Initializes the RPC infrastructure and starts the RPC listenner.
+//
+DWORD RpcInit() {
+ RPC_STATUS status;
+ DWORD dwError;
+
+ status = RpcServerUseProtseqIf(SVCBINDING,
+ RPC_C_LISTEN_MAX_CALLS_DEFAULT,
+ HadoopWinutilSvc_v1_0_s_ifspec,
+ NULL);
+ if (RPC_S_OK != status) {
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ status, L"RpcServerUseProtseqIf");
+ SvcError(status);
+ dwError = status;
+ goto done;
+ }
+
+ status = RpcServerRegisterIfEx(HadoopWinutilSvc_v1_0_s_ifspec,
+ NULL, // MgrTypeUuid
+ NULL, // MgrEpv
+ RPC_IF_ALLOW_LOCAL_ONLY, // Flags
+ RPC_C_LISTEN_MAX_CALLS_DEFAULT, // Max calls
+ RpcAuthorizeCallback); // Auth callback
+
+ if (RPC_S_OK != status) {
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ status, L"RpcServerRegisterIfEx");
+ SvcError(status);
+ dwError = status;
+ goto done;
+ }
+
+ status = RpcServerListen(1, RPC_C_LISTEN_MAX_CALLS_DEFAULT, TRUE);
+ if (RPC_S_ALREADY_LISTENING == status) {
+ ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY,
+ status, L"RpcServerListen");
+ }
+ else if (RPC_S_OK != status) {
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ status, L"RpcServerListen");
+ SvcError(status);
+ dwError = status;
+ goto done;
+ }
+
+ isListenning = TRUE;
+
+ ReportSvcMessage(EVENTLOG_INFORMATION_TYPE, SERVICE_CATEGORY,
+ MSG_RPC_SERVICE_HAS_STARTED);
+
+done:
+ return dwError;
+}
+
+//----------------------------------------------------------------------------
+// Function: RpcStop
+//
+// Description:
+// Tears down the RPC infrastructure and stops the RPC listenner.
+//
+VOID RpcStop() {
+ RPC_STATUS status;
+
+ if (isListenning) {
+
+ status = RpcMgmtStopServerListening(NULL);
+ isListenning = FALSE;
+
+ if (RPC_S_OK != status) {
+ ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY,
+ status, L"RpcMgmtStopServerListening");
+ }
+
+ ReportSvcMessage(EVENTLOG_INFORMATION_TYPE, SERVICE_CATEGORY,
+ MSG_RPC_SERVICE_HAS_STOPPED);
+ }
+}
+
+//----------------------------------------------------------------------------
+// Function: CleanupHandles
+//
+// Description:
+// Cleans up the global service handles.
+//
+VOID CleanupHandles() {
+ if (INVALID_HANDLE_VALUE != ghWaitObject) {
+ UnregisterWait(ghWaitObject);
+ ghWaitObject = INVALID_HANDLE_VALUE;
+ }
+ if (INVALID_HANDLE_VALUE != ghSvcStopEvent) {
+ CloseHandle(ghSvcStopEvent);
+ ghSvcStopEvent = INVALID_HANDLE_VALUE;
+ }
+ if (INVALID_HANDLE_VALUE != ghEventLog) {
+ DeregisterEventSource(ghEventLog);
+ ghEventLog = INVALID_HANDLE_VALUE;
+ }
+}
+
+//----------------------------------------------------------------------------
+// Function: SvcError
+//
+// Description:
+// Aborts the startup sequence. Reports error, stops RPC, cleans up globals.
+//
+VOID SvcError(DWORD dwError) {
+ RpcStop();
+ CleanupHandles();
+ ReportSvcStatus( SERVICE_STOPPED, dwError, 0 );
+}
+
+//----------------------------------------------------------------------------
+// Function: SvcShutdown
+//
+// Description:
+// Callback when the shutdown event is signaled. Stops RPC, cleans up globals.
+//
+VOID CALLBACK SvcShutdown(
+ _In_ PVOID lpParameter,
+ _In_ BOOLEAN TimerOrWaitFired) {
+ RpcStop();
+ CleanupHandles();
+ ReportSvcStatus( SERVICE_STOPPED, NO_ERROR, 0 );
+}
+
+//----------------------------------------------------------------------------
+// Function: SvcCtrlHandler
+//
+// Description:
+// Callback from SCM for for service events (signals).
+//
+// Notes:
+// Shutdown is indirect, we set her the STOP_PENDING state and signal the stop event.
+// Signaling the event invokes SvcShutdown which completes the shutdown.
+// This two staged approach allows the SCM handler to complete fast,
+// not blocking the SCM big fat global lock.
+//
+VOID WINAPI SvcCtrlHandler( DWORD dwCtrl )
+{
+ // Handle the requested control code.
+
+ switch(dwCtrl)
+ {
+ case SERVICE_CONTROL_STOP:
+ ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0);
+
+ // Signal the service to stop.
+ SetEvent(ghSvcStopEvent);
+
+ return;
+
+ default:
+ break;
+ }
+
+}
+
+//----------------------------------------------------------------------------
+// Function: ReportSvcStatus
+//
+// Description:
+// Updates the service status with the SCM.
+//
+VOID ReportSvcStatus( DWORD dwCurrentState,
+ DWORD dwWin32ExitCode,
+ DWORD dwWaitHint)
+{
+ static DWORD dwCheckPoint = 1;
+ DWORD dwError;
+
+ // Fill in the SERVICE_STATUS structure.
+
+ gSvcStatus.dwCurrentState = dwCurrentState;
+ gSvcStatus.dwWin32ExitCode = dwWin32ExitCode;
+ gSvcStatus.dwWaitHint = dwWaitHint;
+
+ if (dwCurrentState == SERVICE_START_PENDING)
+ gSvcStatus.dwControlsAccepted = 0;
+ else gSvcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP;
+
+ if ( (dwCurrentState == SERVICE_RUNNING) ||
+ (dwCurrentState == SERVICE_STOPPED) )
+ gSvcStatus.dwCheckPoint = 0;
+ else gSvcStatus.dwCheckPoint = dwCheckPoint++;
+
+ // Report the status of the service to the SCM.
+ if (!SetServiceStatus( gSvcStatusHandle, &gSvcStatus)) {
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY,
+ dwError, L"SetServiceStatus");
+ };
+}
+
+//----------------------------------------------------------------------------
+// Function: WinutilsCreateProcessAsUser
+//
+// Description:
+// The RPC midl declared function implementation
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// Error code otherwise: otherwise
+//
+// Notes:
+// This is the entry point when the NodeManager does the RPC call
+// Note that the RPC call does not do any S4U work. Is simply spawns (suspended) wintutils
+// using the right command line and the handles over the spwaned process to the NM
+// The actual S4U work occurs in the spawned process, run and monitored by the NM
+//
+error_status_t WinutilsCreateProcessAsUser(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ int nmPid,
+ /* [in] */ CREATE_PROCESS_REQUEST *request,
+ /* [out] */ CREATE_PROCESS_RESPONSE **response) {
+
+ DWORD dwError = ERROR_SUCCESS;
+ LPCWSTR inserts[] = {request->cwd, request->jobName, request->user, request->pidFile, request->cmdLine, NULL};
+ WCHAR winutilsPath[MAX_PATH];
+ WCHAR fullCmdLine[32768];
+ HANDLE taskStdInRd = INVALID_HANDLE_VALUE, taskStdInWr = INVALID_HANDLE_VALUE,
+ taskStdOutRd = INVALID_HANDLE_VALUE, taskStdOutWr = INVALID_HANDLE_VALUE,
+ taskStdErrRd = INVALID_HANDLE_VALUE, taskStdErrWr = INVALID_HANDLE_VALUE,
+ hNmProcess = INVALID_HANDLE_VALUE,
+ hDuplicateProcess = INVALID_HANDLE_VALUE,
+ hDuplicateThread = INVALID_HANDLE_VALUE,
+ hDuplicateStdIn = INVALID_HANDLE_VALUE,
+ hDuplicateStdOut = INVALID_HANDLE_VALUE,
+ hDuplicateStdErr = INVALID_HANDLE_VALUE,
+ hSelfProcess = INVALID_HANDLE_VALUE;
+ BOOL fMustCleanupProcess = FALSE;
+
+ HRESULT hr;
+ STARTUPINFO si;
+ PROCESS_INFORMATION pi;
+ SECURITY_ATTRIBUTES saTaskStdInOutErr;
+
+ ZeroMemory( &si, sizeof(si) );
+ si.cb = sizeof(si);
+ ZeroMemory( &pi, sizeof(pi) );
+ pi.hProcess = INVALID_HANDLE_VALUE;
+ pi.hThread = INVALID_HANDLE_VALUE;
+ ZeroMemory( &saTaskStdInOutErr, sizeof(saTaskStdInOutErr));
+
+ // NB: GetCurrentProcess returns a pseudo-handle that just so happens
+ // has the value -1, ie. INVALID_HANDLE_VALUE. It cannot fail.
+ //
+ hSelfProcess = GetCurrentProcess();
+
+ hNmProcess = OpenProcess(PROCESS_DUP_HANDLE, FALSE, nmPid);
+ if (NULL == hNmProcess) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ GetModuleFileName(NULL, winutilsPath, sizeof(winutilsPath)/sizeof(WCHAR));
+ dwError = GetLastError(); // Always check after GetModuleFileName for ERROR_INSSUFICIENT_BUFFER
+ if (dwError) {
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"GetModuleFileName");
+ goto done;
+ }
+
+ // NB. We can call CreateProcess("wintuls","task create ...") or we can call
+ // CreateProcess(NULL, "winutils task create"). Only the second form passes "task" as
+ // argv[1], as expected by main. First form passes "task" as argv[0] and main fails.
+
+ hr = StringCbPrintf(fullCmdLine, sizeof(fullCmdLine), L"\"%s\" task createAsUser %ls %ls %ls %ls",
+ winutilsPath,
+ request->jobName, request->user, request->pidFile, request->cmdLine);
+ if (FAILED(hr)) {
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ hr, L"StringCbPrintf:fullCmdLine");
+ goto done;
+ }
+
+ LogDebugMessage(L"[%ls]: %ls %ls\n", request->cwd, winutilsPath, fullCmdLine);
+
+ // stdin/stdout/stderr redirection is handled here
+ // We create 3 anonimous named pipes.
+ // Security attributes are required so that the handles can be inherited.
+ // We assign one end of the pipe to the process (stdin gets a read end, stdout gets a write end)
+ // We then duplicate the other end in the NM process, and we close our own handle
+ // Finally we return the duplicate handle values to the NM
+ // The NM will attach Java file dscriptors to the duplicated handles and
+ // read/write them as ordinary Java InputStream/OutputStream objects
+
+ si.dwFlags |= STARTF_USESTDHANDLES;
+
+ saTaskStdInOutErr.nLength = sizeof(SECURITY_ATTRIBUTES);
+ saTaskStdInOutErr.bInheritHandle = TRUE;
+ saTaskStdInOutErr.lpSecurityDescriptor = NULL;
+
+ if (!CreatePipe(&taskStdInRd, &taskStdInWr, &saTaskStdInOutErr, 0)) {
+ dwError = GetLastError();
+ goto done;
+ }
+ if (!SetHandleInformation(taskStdInWr, HANDLE_FLAG_INHERIT, FALSE)) {
+ dwError = GetLastError();
+ goto done;
+ }
+ si.hStdInput = taskStdInRd;
+
+ if (!CreatePipe(&taskStdOutRd, &taskStdOutWr, &saTaskStdInOutErr, 0)) {
+ dwError = GetLastError();
+ goto done;
+ }
+ if (!SetHandleInformation(taskStdOutRd, HANDLE_FLAG_INHERIT, FALSE)) {
+ dwError = GetLastError();
+ goto done;
+ }
+ si.hStdOutput = taskStdOutWr;
+
+ if (!CreatePipe(&taskStdErrRd, &taskStdErrWr, &saTaskStdInOutErr, 0)) {
+ dwError = GetLastError();
+ goto done;
+ }
+ if (!SetHandleInformation(taskStdErrRd, HANDLE_FLAG_INHERIT, FALSE)) {
+ dwError = GetLastError();
+ goto done;
+ }
+ si.hStdError = taskStdErrWr;
+
+ if (!CreateProcess(
+ NULL, // lpApplicationName,
+ fullCmdLine, // lpCommandLine,
+ NULL, // lpProcessAttributes,
+ NULL, // lpThreadAttributes,
+ TRUE, // bInheritHandles,
+ CREATE_SUSPENDED, // dwCreationFlags,
+ NULL, // lpEnvironment,
+ request->cwd, // lpCurrentDirectory,
+ &si, // lpStartupInfo
+ &pi)) { // lpProcessInformation
+
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"CreateProcess");
+ goto done;
+ }
+
+ fMustCleanupProcess = TRUE;
+
+ LogDebugMessage(L"CreateProcess: pid:%x\n", pi.dwProcessId);
+
+ if (!DuplicateHandle(hSelfProcess, pi.hProcess, hNmProcess,
+ &hDuplicateProcess, 0, FALSE, DUPLICATE_SAME_ACCESS)) {
+ dwError = GetLastError();
+ LogDebugMessage(L"failed: pi.hProcess\n");
+ goto done;
+ }
+
+ if (!DuplicateHandle(hSelfProcess, pi.hThread, hNmProcess,
+ &hDuplicateThread, 0, FALSE, DUPLICATE_SAME_ACCESS)) {
+ dwError = GetLastError();
+ LogDebugMessage(L"failed: pi.hThread\n");
+ goto done;
+ }
+
+ if (!DuplicateHandle(hSelfProcess, taskStdInWr, hNmProcess,
+ &hDuplicateStdIn, 0, FALSE, DUPLICATE_SAME_ACCESS)) {
+ dwError = GetLastError();
+ LogDebugMessage(L"failed: taskStdInWr\n");
+ goto done;
+ }
+
+ if (!DuplicateHandle(hSelfProcess, taskStdOutRd, hNmProcess,
+ &hDuplicateStdOut, 0, FALSE, DUPLICATE_SAME_ACCESS)) {
+ dwError = GetLastError();
+ LogDebugMessage(L"failed: taskStdOutRd\n");
+ goto done;
+ }
+
+ if (!DuplicateHandle(hSelfProcess, taskStdErrRd, hNmProcess,
+ &hDuplicateStdErr, 0, FALSE, DUPLICATE_SAME_ACCESS)) {
+ dwError = GetLastError();
+ LogDebugMessage(L"failed: taskStdErrRd\n");
+ goto done;
+ }
+
+ *response = (CREATE_PROCESS_RESPONSE*) MIDL_user_allocate(sizeof(CREATE_PROCESS_RESPONSE));
+ if (NULL == *response) {
+ dwError = ERROR_OUTOFMEMORY;
+ LogDebugMessage(L"Failed to allocate CREATE_PROCESS_RESPONSE* response\n");
+ goto done;
+ }
+
+ // We're now transfering ownership of the duplicated handles to the caller
+ // If the RPC call fails *after* this point the handles are leaked inside the NM process
+
+ (*response)->hProcess = hDuplicateProcess;
+ (*response)->hThread = hDuplicateThread;
+ (*response)->hStdIn = hDuplicateStdIn;
+ (*response)->hStdOut = hDuplicateStdOut;
+ (*response)->hStdErr = hDuplicateStdErr;
+
+ fMustCleanupProcess = FALSE;
+
+done:
+
+ if (fMustCleanupProcess) {
+ LogDebugMessage(L"Cleaning process: %d due to error:%d\n", pi.dwProcessId, dwError);
+ TerminateProcess(pi.hProcess, EXIT_FAILURE);
+
+ // cleanup the duplicate handles inside the NM.
+
+ if (INVALID_HANDLE_VALUE != hDuplicateProcess) {
+ DuplicateHandle(hNmProcess, hDuplicateProcess, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE);
+ }
+ if (INVALID_HANDLE_VALUE != hDuplicateThread) {
+ DuplicateHandle(hNmProcess, hDuplicateThread, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE);
+ }
+ if (INVALID_HANDLE_VALUE != hDuplicateStdIn) {
+ DuplicateHandle(hNmProcess, hDuplicateStdIn, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE);
+ }
+ if (INVALID_HANDLE_VALUE != hDuplicateStdOut) {
+ DuplicateHandle(hNmProcess, hDuplicateStdOut, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE);
+ }
+ if (INVALID_HANDLE_VALUE != hDuplicateStdErr) {
+ DuplicateHandle(hNmProcess, hDuplicateStdErr, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE);
+ }
+ }
+
+ if (INVALID_HANDLE_VALUE != hSelfProcess) CloseHandle(hSelfProcess);
+ if (INVALID_HANDLE_VALUE != hNmProcess) CloseHandle(hNmProcess);
+ if (INVALID_HANDLE_VALUE != taskStdInRd) CloseHandle(taskStdInRd);
+ if (INVALID_HANDLE_VALUE != taskStdInWr) CloseHandle(taskStdInWr);
+ if (INVALID_HANDLE_VALUE != taskStdOutRd) CloseHandle(taskStdOutRd);
+ if (INVALID_HANDLE_VALUE != taskStdOutWr) CloseHandle(taskStdOutWr);
+ if (INVALID_HANDLE_VALUE != taskStdErrRd) CloseHandle(taskStdErrRd);
+ if (INVALID_HANDLE_VALUE != taskStdErrWr) CloseHandle(taskStdErrWr);
+
+
+ // This is closing our own process/thread handles.
+ // If the transfer was succesfull the NM has its own duplicates (if any)
+ if (INVALID_HANDLE_VALUE != pi.hThread) CloseHandle(pi.hThread);
+ if (INVALID_HANDLE_VALUE != pi.hProcess) CloseHandle(pi.hProcess);
+
+ return dwError;
+}
+
+error_status_t WinutilsCreateFile(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ int nm_pid,
+ /* [in] */ CREATEFILE_REQUEST *request,
+ /* [out] */ CREATEFILE_RESPONSE **response) {
+
+ DWORD dwError = ERROR_SUCCESS;
+
+ HANDLE hNmProcess = INVALID_HANDLE_VALUE,
+ hFile = INVALID_HANDLE_VALUE,
+ hDuplicateFile = INVALID_HANDLE_VALUE,
+ hSelfProcess = GetCurrentProcess();
+
+ SECURITY_ATTRIBUTES saFile;
+
+ ZeroMemory( &saFile, sizeof(saFile));
+
+ saFile.nLength = sizeof(SECURITY_ATTRIBUTES);
+ saFile.bInheritHandle = TRUE;
+ saFile.lpSecurityDescriptor = NULL;
+
+ hFile = CreateFile(
+ request->path,
+ request->desiredAccess,
+ request->shareMode,
+ &saFile,
+ request->creationDisposition,
+ request->flags,
+ NULL); // hTemplate
+ if (INVALID_HANDLE_VALUE == hFile) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ hNmProcess = OpenProcess(PROCESS_DUP_HANDLE, FALSE, nm_pid);
+ if (NULL == hNmProcess) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ if (!DuplicateHandle(hSelfProcess, hFile,
+ hNmProcess, &hDuplicateFile,
+ 0, FALSE, DUPLICATE_SAME_ACCESS)) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ *response = (CREATEFILE_RESPONSE*) MIDL_user_allocate(sizeof(CREATEFILE_RESPONSE));
+ if (NULL == *response) {
+ dwError = ERROR_OUTOFMEMORY;
+ goto done;
+ }
+
+ (*response)->hFile = hDuplicateFile;
+ hDuplicateFile = INVALID_HANDLE_VALUE;
+
+done:
+
+ if (INVALID_HANDLE_VALUE != hFile) CloseHandle(hFile);
+ if (INVALID_HANDLE_VALUE != hDuplicateFile) {
+ DuplicateHandle(hNmProcess, hDuplicateFile, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE);
+ }
+ if (INVALID_HANDLE_VALUE != hNmProcess) CloseHandle(hNmProcess);
+
+ LogDebugMessage(L"WinutilsCreateFile: %s %d, %d, %d, %d: %d",
+ request->path,
+ request->desiredAccess,
+ request->shareMode,
+ request->creationDisposition,
+ request->flags,
+ dwError);
+
+ return dwError;
+}
+
+error_status_t WinutilsMkDir(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ MKDIR_REQUEST *request) {
+ DWORD dwError = ERROR_SUCCESS;
+ if (!CreateDirectory(request->filePath, NULL)) {
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"CreateDirectory");
+ }
+ LogDebugMessage(L"WinutilsMkDir: %s :%d\n", request->filePath, dwError);
+ return dwError;
+}
+
+error_status_t WinutilsChown(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ CHOWN_REQUEST *request) {
+ DWORD dwError = ERROR_SUCCESS;
+ dwError = ChownImpl(request->ownerName, request->groupName, request->filePath);
+ if (dwError) {
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"ChownImpl");
+ }
+ LogDebugMessage(L"WinutilsChown: %s %s %s :%d\n",
+ request->ownerName, request->groupName, request->filePath, dwError);
+ return dwError;
+}
+
+error_status_t WinutilsChmod(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ CHMOD_REQUEST *request) {
+ DWORD dwError = ERROR_SUCCESS;
+ dwError = ChangeFileModeByMask(request->filePath, request->mode);
+ if (dwError) {
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"ChangeFileModeByMask");
+ }
+ LogDebugMessage(L"WinutilsChmod: %s %o :%d\n",
+ request->filePath, request->mode, dwError);
+ return dwError;
+}
+
+error_status_t WinutilsMoveFile(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ MOVEFILE_REQUEST *request) {
+ DWORD dwError = ERROR_SUCCESS;
+ DWORD flags = 0;
+
+ switch (request->operation) {
+ case MOVE_FILE:
+ flags |= MOVEFILE_COPY_ALLOWED;
+ if (request->replaceExisting) flags |= MOVEFILE_REPLACE_EXISTING;
+ if (!MoveFileEx(request->sourcePath, request->destinationPath, flags)) {
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"MoveFileEx");
+ }
+ break;
+ case COPY_FILE:
+ if (!request->replaceExisting) flags |= COPY_FILE_FAIL_IF_EXISTS;
+ if (!CopyFileEx(request->sourcePath, request->destinationPath,
+ NULL, // lpProgressRoutine
+ NULL, // lpData
+ NULL, // pbCancel
+ flags)) {
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"CopyFileEx");
+ }
+ }
+ LogDebugMessage(L"WinutilsMoveFile: %d: %s %s :%d\n",
+ request->operation, request->sourcePath, request->destinationPath, dwError);
+ return dwError;
+}
+
+
+//----------------------------------------------------------------------------
+// Function: ServiceUsage
+//
+// Description:
+// Prints the CLI arguments for service command.
+//
+void ServiceUsage()
+{
+ fwprintf(stdout, L"\
+ Usage: service\n\
+ Starts the nodemanager Windows Secure Container Executor helper service.\n\
+ The service must run as a high privileged account (LocalSystem)\n\
+ and is used by the nodemanager WSCE to spawn secure containers on Windows.\n");
+}
+
+
diff --git hadoop-common-project/hadoop-common/src/main/winutils/symlink.c hadoop-common-project/hadoop-common/src/main/winutils/symlink.c
index ea372cc..02acd4d 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/symlink.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/symlink.c
@@ -77,7 +77,7 @@ int Symlink(__in int argc, __in_ecount(argc) wchar_t *argv[])
// This is just an additional step to do the privilege check by not using
// error code from CreateSymbolicLink() method.
//
- if (!EnablePrivilege(L"SeCreateSymbolicLinkPrivilege"))
+ if (EnablePrivilege(L"SeCreateSymbolicLinkPrivilege") != ERROR_SUCCESS)
{
fwprintf(stderr,
L"No privilege to create symbolic links.\n");
diff --git hadoop-common-project/hadoop-common/src/main/winutils/task.c hadoop-common-project/hadoop-common/src/main/winutils/task.c
index 19bda96..67678f2 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/task.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/task.c
@@ -18,15 +18,23 @@
#include "winutils.h"
#include
#include
+#include
+#include
#define PSAPI_VERSION 1
#pragma comment(lib, "psapi.lib")
-#define ERROR_TASK_NOT_ALIVE 1
+#define NM_WSCE_IMPERSONATE_ALLOWED L"yarn.nodemanager.windows-secure-container-executor.impersonate.allowed"
+#define NM_WSCE_IMPERSONATE_DENIED L"yarn.nodemanager.windows-secure-container-executor.impersonate.denied"
-// This exit code for killed processes is compatible with Unix, where a killed
-// process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137.
-#define KILLED_PROCESS_EXIT_CODE 137
+// The S4U impersonation access check mask. Arbitrary value (we use 1 for the service access check)
+#define SERVICE_IMPERSONATE_MASK 0x00000002
+
+
+// Name for tracking this logon process when registering with LSA
+static const char *LOGON_PROCESS_NAME="Hadoop Container Executor";
+// Name for token source, must be less or eq to TOKEN_SOURCE_LENGTH (currently 8) chars
+static const char *TOKEN_SOURCE_NAME = "HadoopEx";
// List of different task related command line options supported by
// winutils.
@@ -34,6 +42,7 @@ typedef enum TaskCommandOptionType
{
TaskInvalid,
TaskCreate,
+ TaskCreateAsUser,
TaskIsAlive,
TaskKill,
TaskProcessList
@@ -86,37 +95,393 @@ static BOOL ParseCommandLine(__in int argc,
}
}
+ if (argc >= 6) {
+ if (wcscmp(argv[1], L"createAsUser") == 0)
+ {
+ *command = TaskCreateAsUser;
+ return TRUE;
+ }
+ }
+
return FALSE;
}
+
//----------------------------------------------------------------------------
-// Function: createTask
+// Function: BuildImpersonateSecurityDescriptor
+//
+// Description:
+// Builds the security descriptor for the S4U impersonation permissions
+// This describes what users can be impersonated and what not
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+//
+DWORD BuildImpersonateSecurityDescriptor(__out PSECURITY_DESCRIPTOR* ppSD) {
+ DWORD dwError = ERROR_SUCCESS;
+ size_t countAllowed = 0;
+ PSID* allowedSids = NULL;
+ size_t countDenied = 0;
+ PSID* deniedSids = NULL;
+ LPCWSTR value = NULL;
+ WCHAR** tokens = NULL;
+ size_t len = 0;
+ size_t count = 0;
+ int crt = 0;
+ PSECURITY_DESCRIPTOR pSD = NULL;
+
+ dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_IMPERSONATE_ALLOWED, &len, &value);
+ if (dwError) {
+ ReportErrorCode(L"GetConfigValue:1", dwError);
+ goto done;
+ }
+
+ if (0 == len) {
+ dwError = ERROR_BAD_CONFIGURATION;
+ ReportErrorCode(L"GetConfigValue:2", dwError);
+ goto done;
+ }
+
+ dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens);
+ if (dwError) {
+ ReportErrorCode(L"SplitStringIgnoreSpaceW:1", dwError);
+ goto done;
+ }
+
+ allowedSids = LocalAlloc(LPTR, sizeof(PSID) * count);
+ if (NULL == allowedSids) {
+ dwError = GetLastError();
+ ReportErrorCode(L"LocalAlloc:1", dwError);
+ goto done;
+ }
+
+ for(crt = 0; crt < count; ++crt) {
+ dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]);
+ if (dwError) {
+ ReportErrorCode(L"GetSidFromAcctNameW:1", dwError);
+ goto done;
+ }
+ }
+ countAllowed = count;
+
+ LocalFree(tokens);
+ tokens = NULL;
+
+ LocalFree(value);
+ value = NULL;
+
+ dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_IMPERSONATE_DENIED, &len, &value);
+ if (dwError) {
+ ReportErrorCode(L"GetConfigValue:3", dwError);
+ goto done;
+ }
+
+ if (0 != len) {
+ dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens);
+ if (dwError) {
+ ReportErrorCode(L"SplitStringIgnoreSpaceW:2", dwError);
+ goto done;
+ }
+
+ deniedSids = LocalAlloc(LPTR, sizeof(PSID) * count);
+ if (NULL == allowedSids) {
+ dwError = GetLastError();
+ ReportErrorCode(L"LocalAlloc:2", dwError);
+ goto done;
+ }
+
+ for(crt = 0; crt < count; ++crt) {
+ dwError = GetSidFromAcctNameW(tokens[crt], &deniedSids[crt]);
+ if (dwError) {
+ ReportErrorCode(L"GetSidFromAcctNameW:2", dwError);
+ goto done;
+ }
+ }
+ countDenied = count;
+ }
+
+ dwError = BuildServiceSecurityDescriptor(
+ SERVICE_IMPERSONATE_MASK,
+ countAllowed, allowedSids,
+ countDenied, deniedSids,
+ &pSD);
+
+ if (dwError) {
+ ReportErrorCode(L"BuildServiceSecurityDescriptor", dwError);
+ goto done;
+ }
+
+ *ppSD = pSD;
+ pSD = NULL;
+
+done:
+ if (pSD) LocalFree(pSD);
+ if (tokens) LocalFree(tokens);
+ if (allowedSids) LocalFree(allowedSids);
+ if (deniedSids) LocalFree(deniedSids);
+ return dwError;
+}
+
+//----------------------------------------------------------------------------
+// Function: BuildJobObjectSecurityDescriptor
+//
+// Description:
+// Builds the security descriptor for NT job object that contains the task
+// Both the nodemanager and the container user require access to the job object
+// The ACEs grant full controll to NM, container job and LocalSystem (the WSCE winutils service)
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+//
+DWORD BuildJobObjectSecurityDescriptor(
+ __in LPCWSTR user,
+ __out PSECURITY_DESCRIPTOR* pSD) {
+
+ DWORD dwError;
+
+ DWORD cbSid = SECURITY_MAX_SID_SIZE;
+ PSID pSidNodeManager = NULL;
+ PSID pSidLocalSystem = NULL;
+ PSID pSidUser = NULL;
+ PSID* allowedSids = NULL;
+ int countSids = 0;
+ int countTokens = 0;
+ int len = 0;
+ LPCWSTR value = NULL;
+ WCHAR** tokens = NULL;
+ int crt = 0;
+
+ dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_ALLOWED, &len, &value);
+ if (ERROR_SUCCESS != dwError) {
+ ReportErrorCode(L"GetConfigValue", dwError);
+ goto done;
+ }
+
+ if (0 == len) {
+ dwError = ERROR_BAD_CONFIGURATION;
+ ReportErrorCode(L"GetConfigValue", dwError);
+ goto done;
+ }
+
+ dwError = SplitStringIgnoreSpaceW(len, value, L',', &countTokens, &tokens);
+ if (ERROR_SUCCESS != dwError) {
+ ReportErrorCode(L"SplitStringIgnoreSpaceW", dwError);
+ goto done;
+ }
+
+ // allocate for all the configure granted users (usually NM service account)
+ // +1 for the container user
+ // +1 for LocalSystem
+ //
+ allowedSids = (PSID*) LocalAlloc(LPTR, sizeof(PSID) * (countTokens + 2));
+ if (NULL == allowedSids) {
+ dwError = ERROR_OUTOFMEMORY;
+ ReportErrorCode(L"LocalAlloc:pSidLocalSystem", dwError);
+ goto done;
+ }
+
+ for (crt = 0; crt < countTokens; ++crt) {
+ dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]);
+ if (ERROR_SUCCESS != dwError) {
+ ReportErrorCode(L"GetSidFromAcctNameW", dwError);
+ goto done;
+ }
+ }
+
+ dwError = GetSidFromAcctNameW(user, &allowedSids[crt]);
+ if (ERROR_SUCCESS != dwError) {
+ ReportErrorCode(L"GetSidFromAcctNameW:user", dwError);
+ goto done;
+ }
+
+ ++crt;
+
+ allowedSids[crt] = (PSID) LocalAlloc(LPTR, SECURITY_MAX_SID_SIZE);
+ if (NULL == allowedSids[crt]) {
+ dwError = ERROR_OUTOFMEMORY;
+ ReportErrorCode(L"LocalAlloc:pSidLocalSystem", dwError);
+ goto done;
+ }
+ cbSid = SECURITY_MAX_SID_SIZE;
+ if (!CreateWellKnownSid(WinLocalSystemSid, NULL, allowedSids[crt], &cbSid)) {
+ dwError = GetLastError();
+ ReportErrorCode(L"CreateWellKnownSid", dwError);
+ goto done;
+ }
+
+ dwError = BuildServiceSecurityDescriptor(JOB_OBJECT_ALL_ACCESS, crt, allowedSids, 0, NULL, pSD);
+ if (ERROR_SUCCESS != dwError) {
+ goto done;
+ }
+
+done:
+ do {
+ if (allowedSids && allowedSids[crt]) LocalFree(allowedSids[crt]);
+ --crt;
+ } while (crt);
+ if (allowedSids) LocalFree(allowedSids);
+ if (value) LocalFree(value);
+
+ return dwError;
+}
+
+
+
+//----------------------------------------------------------------------------
+// Function: ValidateImpersonateAccessCheck
+//
+// Description:
+// Performs the access check for S4U impersonation
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// ERROR_ACCESS_DENIED, GetLastError: otherwise
+//
+DWORD ValidateImpersonateAccessCheck(__in HANDLE logonHandle) {
+ DWORD dwError = ERROR_SUCCESS;
+ PSECURITY_DESCRIPTOR pSD = NULL;
+ LUID luidUnused;
+ AUTHZ_ACCESS_REQUEST request;
+ AUTHZ_ACCESS_REPLY reply;
+ DWORD authError = ERROR_SUCCESS;
+ DWORD saclResult = 0;
+ ACCESS_MASK grantedMask = 0;
+ AUTHZ_RESOURCE_MANAGER_HANDLE hManager = NULL;
+ AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzToken = NULL;
+
+ ZeroMemory(&luidUnused, sizeof(luidUnused));
+ ZeroMemory(&request, sizeof(request));
+ ZeroMemory(&reply, sizeof(reply));
+
+ dwError = BuildImpersonateSecurityDescriptor(&pSD);
+ if (dwError) {
+ ReportErrorCode(L"BuildImpersonateSecurityDescriptor", dwError);
+ goto done;
+ }
+
+ request.DesiredAccess = MAXIMUM_ALLOWED;
+ reply.Error = &authError;
+ reply.SaclEvaluationResults = &saclResult;
+ reply.ResultListLength = 1;
+ reply.GrantedAccessMask = &grantedMask;
+
+ if (!AuthzInitializeResourceManager(
+ AUTHZ_RM_FLAG_NO_AUDIT,
+ NULL, // pfnAccessCheck
+ NULL, // pfnComputeDynamicGroups
+ NULL, // pfnFreeDynamicGroups
+ NULL, // szResourceManagerName
+ &hManager)) {
+ dwError = GetLastError();
+ ReportErrorCode(L"AuthzInitializeResourceManager", dwError);
+ goto done;
+ }
+
+ if (!AuthzInitializeContextFromToken(
+ 0,
+ logonHandle,
+ hManager,
+ NULL, // expiration time
+ luidUnused, // not used
+ NULL, // callback args
+ &hAuthzToken)) {
+ dwError = GetLastError();
+ ReportErrorCode(L"AuthzInitializeContextFromToken", dwError);
+ goto done;
+ }
+
+ if (!AuthzAccessCheck(
+ 0,
+ hAuthzToken,
+ &request,
+ NULL, // AuditEvent
+ pSD,
+ NULL, // OptionalSecurityDescriptorArray
+ 0, // OptionalSecurityDescriptorCount
+ &reply,
+ NULL // phAccessCheckResults
+ )) {
+ dwError = GetLastError();
+ ReportErrorCode(L"AuthzAccessCheck", dwError);
+ goto done;
+ }
+
+ LogDebugMessage(L"AutzAccessCheck: Error:%d sacl:%d access:%d\n",
+ authError, saclResult, grantedMask);
+
+ if (authError != ERROR_SUCCESS) {
+ ReportErrorCode(L"AuthzAccessCheck:REPLY:1", authError);
+ dwError = authError;
+ }
+ else if (!(grantedMask & SERVICE_IMPERSONATE_MASK)) {
+ ReportErrorCode(L"AuthzAccessCheck:REPLY:2", ERROR_ACCESS_DENIED);
+ dwError = ERROR_ACCESS_DENIED;
+ }
+
+done:
+ if (hAuthzToken) AuthzFreeContext(hAuthzToken);
+ if (hManager) AuthzFreeResourceManager(hManager);
+ if (pSD) LocalFree(pSD);
+ return dwError;
+}
+
+//----------------------------------------------------------------------------
+// Function: CreateTaskImpl
//
// Description:
// Creates a task via a jobobject. Outputs the
// appropriate information to stdout on success, or stderr on failure.
+// logonHandle may be NULL, in this case the current logon will be utilized for the
+// created process
//
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
-DWORD createTask(__in PCWSTR jobObjName,__in PWSTR cmdLine)
+DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PCWSTR cmdLine,
+ __in SECURITY_DESCRIPTOR* pSdJob)
{
- DWORD err = ERROR_SUCCESS;
+ DWORD dwErrorCode = ERROR_SUCCESS;
DWORD exitCode = EXIT_FAILURE;
+ DWORD currDirCnt = 0;
STARTUPINFO si;
PROCESS_INFORMATION pi;
HANDLE jobObject = NULL;
JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = { 0 };
+ void * envBlock = NULL;
+ BOOL createProcessResult = FALSE;
+ SECURITY_ATTRIBUTES saJob;
+ SECURITY_ATTRIBUTES* psaJob = NULL;
+
+ wchar_t* curr_dir = NULL;
+ FILE *stream = NULL;
+
+ if (NULL != logonHandle) {
+ dwErrorCode = ValidateImpersonateAccessCheck(logonHandle);
+ if (dwErrorCode) {
+ ReportErrorCode(L"ValidateImpersonateAccessCheck", dwErrorCode);
+ return dwErrorCode;
+ }
+
+ // We need to create a security descripto for the job so that the task can query it
+ ZeroMemory(&saJob, sizeof(saJob));
+
+ saJob.nLength = sizeof(saJob);
+ saJob.lpSecurityDescriptor = pSdJob;
+ psaJob = &saJob;
+ }
// Create un-inheritable job object handle and set job object to terminate
// when last handle is closed. So winutils.exe invocation has the only open
// job object handle. Exit of winutils.exe ensures termination of job object.
// Either a clean exit of winutils or crash or external termination.
- jobObject = CreateJobObject(NULL, jobObjName);
- err = GetLastError();
- if(jobObject == NULL || err == ERROR_ALREADY_EXISTS)
+ jobObject = CreateJobObject(psaJob, jobObjName);
+ dwErrorCode = GetLastError();
+ if(jobObject == NULL || dwErrorCode == ERROR_ALREADY_EXISTS)
{
- return err;
+ ReportErrorCode(L"CreateJobObject", dwErrorCode);
+ return dwErrorCode;
}
jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
if(SetInformationJobObject(jobObject,
@@ -124,48 +489,127 @@ DWORD createTask(__in PCWSTR jobObjName,__in PWSTR cmdLine)
&jeli,
sizeof(jeli)) == 0)
{
- err = GetLastError();
+ dwErrorCode = GetLastError();
+ ReportErrorCode(L"SetInformationJobObject", dwErrorCode);
CloseHandle(jobObject);
- return err;
+ return dwErrorCode;
}
if(AssignProcessToJobObject(jobObject, GetCurrentProcess()) == 0)
{
- err = GetLastError();
+ dwErrorCode = GetLastError();
+ ReportErrorCode(L"AssignProcessToJobObject", dwErrorCode);
CloseHandle(jobObject);
- return err;
+ return dwErrorCode;
}
// the child JVM uses this env var to send the task OS process identifier
// to the TaskTracker. We pass the job object name.
if(SetEnvironmentVariable(L"JVM_PID", jobObjName) == 0)
{
- err = GetLastError();
- CloseHandle(jobObject);
- return err;
+ dwErrorCode = GetLastError();
+ ReportErrorCode(L"SetEnvironmentVariable", dwErrorCode);
+ // We have to explictly Terminate, passing in the error code
+ // simply closing the job would kill our own process with success exit status
+ TerminateJobObject(jobObject, dwErrorCode);
+ return dwErrorCode;
}
ZeroMemory( &si, sizeof(si) );
si.cb = sizeof(si);
ZeroMemory( &pi, sizeof(pi) );
- if (CreateProcess(NULL, cmdLine, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi) == 0)
- {
- err = GetLastError();
- CloseHandle(jobObject);
- return err;
+ if( logonHandle != NULL ) {
+ // create user environment for this logon
+ if(!CreateEnvironmentBlock(&envBlock,
+ logonHandle,
+ TRUE )) {
+ dwErrorCode = GetLastError();
+ ReportErrorCode(L"CreateEnvironmentBlock", dwErrorCode);
+ // We have to explictly Terminate, passing in the error code
+ // simply closing the job would kill our own process with success exit status
+ TerminateJobObject(jobObject, dwErrorCode);
+ return dwErrorCode;
+ }
+ }
+
+ // Get the required buffer size first
+ currDirCnt = GetCurrentDirectory(0, NULL);
+ if (0 < currDirCnt) {
+ curr_dir = (wchar_t*) alloca(currDirCnt * sizeof(wchar_t));
+ assert(curr_dir);
+ currDirCnt = GetCurrentDirectory(currDirCnt, curr_dir);
+ }
+
+ if (0 == currDirCnt) {
+ dwErrorCode = GetLastError();
+ ReportErrorCode(L"GetCurrentDirectory", dwErrorCode);
+ // We have to explictly Terminate, passing in the error code
+ // simply closing the job would kill our own process with success exit status
+ TerminateJobObject(jobObject, dwErrorCode);
+ return dwErrorCode;
+ }
+
+ if (logonHandle == NULL) {
+ createProcessResult = CreateProcess(
+ NULL, // ApplicationName
+ cmdLine, // command line
+ NULL, // process security attributes
+ NULL, // thread security attributes
+ TRUE, // inherit handles
+ 0, // creation flags
+ NULL, // environment
+ curr_dir, // current directory
+ &si, // startup info
+ &pi); // process info
+ }
+ else {
+ createProcessResult = CreateProcessAsUser(
+ logonHandle, // logon token handle
+ NULL, // Application handle
+ cmdLine, // command line
+ NULL, // process security attributes
+ NULL, // thread security attributes
+ FALSE, // inherit handles
+ CREATE_UNICODE_ENVIRONMENT, // creation flags
+ envBlock, // environment
+ curr_dir, // current directory
+ &si, // startup info
+ &pi); // process info
+ }
+
+ if (FALSE == createProcessResult) {
+ dwErrorCode = GetLastError();
+ ReportErrorCode(L"CreateProcess/AsUser", dwErrorCode);
+ if( envBlock != NULL ) {
+ DestroyEnvironmentBlock( envBlock );
+ envBlock = NULL;
+ }
+ // We have to explictly Terminate, passing in the error code
+ // simply closing the job would kill our own process with success exit status
+ TerminateJobObject(jobObject, dwErrorCode);
+
+ // This is tehnically dead code, we cannot reach this condition
+ return dwErrorCode;
}
CloseHandle(pi.hThread);
+ ReportErrorCode(L"CreateTaskImpl", ERROR_SUCCESS);
+
// Wait until child process exits.
WaitForSingleObject( pi.hProcess, INFINITE );
if(GetExitCodeProcess(pi.hProcess, &exitCode) == 0)
{
- err = GetLastError();
+ dwErrorCode = GetLastError();
}
CloseHandle( pi.hProcess );
+ if( envBlock != NULL ) {
+ DestroyEnvironmentBlock( envBlock );
+ envBlock = NULL;
+ }
+
// Terminate job object so that all spawned processes are also killed.
// This is needed because once this process closes the handle to the job
// object and none of the spawned objects have the handle open (via
@@ -173,21 +617,163 @@ DWORD createTask(__in PCWSTR jobObjName,__in PWSTR cmdLine)
// program (say winutils task kill) to terminate this job object via its name.
if(TerminateJobObject(jobObject, exitCode) == 0)
{
- err = GetLastError();
+ dwErrorCode = GetLastError();
}
- // comes here only on failure or TerminateJobObject
+ // comes here only on failure of TerminateJobObject
CloseHandle(jobObject);
- if(err != ERROR_SUCCESS)
+ if(dwErrorCode != ERROR_SUCCESS)
{
- return err;
+ return dwErrorCode;
}
return exitCode;
}
//----------------------------------------------------------------------------
-// Function: isTaskAlive
+// Function: CreateTask
+//
+// Description:
+// Creates a task via a jobobject. Outputs the
+// appropriate information to stdout on success, or stderr on failure.
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+DWORD CreateTask(__in PCWSTR jobObjName,__in PWSTR cmdLine)
+{
+ // call with null logon in order to create tasks utilizing the current logon
+ return CreateTaskImpl( NULL, jobObjName, cmdLine, NULL);
+}
+
+//----------------------------------------------------------------------------
+// Function: CreateTaskAsUser
+//
+// Description:
+// Creates a task via a jobobject. Outputs the
+// appropriate information to stdout on success, or stderr on failure.
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+DWORD CreateTaskAsUser(__in PCWSTR jobObjName,
+ __in PCWSTR user, __in PCWSTR pidFilePath, __in PCWSTR cmdLine)
+{
+ DWORD err = ERROR_SUCCESS;
+ DWORD exitCode = EXIT_FAILURE;
+ ULONG authnPkgId;
+ HANDLE lsaHandle = INVALID_HANDLE_VALUE;
+ PROFILEINFO pi;
+ BOOL profileIsLoaded = FALSE;
+ FILE* pidFile = NULL;
+ SECURITY_DESCRIPTOR* pSdJob = NULL;
+ DWORD retLen = 0;
+ HANDLE logonHandle = NULL;
+
+ err = EnablePrivilege(SE_TCB_NAME);
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"EnablePrivilege:SE_TCB_NAME", err);
+ goto done;
+ }
+ err = EnablePrivilege(SE_ASSIGNPRIMARYTOKEN_NAME);
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"EnablePrivilege:SE_ASSIGNPRIMARYTOKEN_NAME", err);
+ goto done;
+ }
+ err = EnablePrivilege(SE_INCREASE_QUOTA_NAME);
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"EnablePrivilege:SE_INCREASE_QUOTA_NAME", err);
+ goto done;
+ }
+ err = EnablePrivilege(SE_RESTORE_NAME);
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"EnablePrivilege:SE_RESTORE_NAME", err);
+ goto done;
+ }
+
+ err = EnablePrivilege(SE_SECURITY_NAME);
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"EnablePrivilege:SE_SECURITY_NAME", err);
+ goto done;
+ }
+
+ err = RegisterWithLsa(LOGON_PROCESS_NAME ,&lsaHandle);
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"RegisterWithLsa", err);
+ goto done;
+ }
+
+ err = LookupKerberosAuthenticationPackageId( lsaHandle, &authnPkgId );
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"LookupKerberosAuthenticationPackageId", err);
+ goto done;
+ }
+
+ err = CreateLogonForUser(lsaHandle,
+ LOGON_PROCESS_NAME,
+ TOKEN_SOURCE_NAME,
+ authnPkgId,
+ user,
+ &logonHandle);
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"CreateLogonForUser", err);
+ goto done;
+ }
+
+ err = LoadUserProfileForLogon(logonHandle, &pi);
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"LoadUserProfileForLogon", err);
+ goto done;
+ }
+ profileIsLoaded = TRUE;
+
+ // Create the PID file
+
+ if (!(pidFile = _wfopen(pidFilePath, "w"))) {
+ err = GetLastError();
+ ReportErrorCode(L"_wfopen:pidFilePath", err);
+ goto done;
+ }
+
+ if (0 > fprintf_s(pidFile, "%ls", jobObjName)) {
+ err = GetLastError();
+ }
+
+ fclose(pidFile);
+
+ if (err != ERROR_SUCCESS) {
+ ReportErrorCode(L"fprintf_s:pidFilePath", err);
+ goto done;
+ }
+
+ err = BuildJobObjectSecurityDescriptor(user, &pSdJob);
+ if (ERROR_SUCCESS != err) {
+ ReportErrorCode(L"BuildJobSecurityDescriptor", err);
+ goto done;
+ }
+
+ err = CreateTaskImpl(logonHandle, jobObjName, cmdLine, pSdJob);
+
+done:
+ if (pSdJob) LocalFree(pSdJob);
+
+ if( profileIsLoaded ) {
+ UnloadProfileForLogon( logonHandle, &pi );
+ profileIsLoaded = FALSE;
+ }
+ if( logonHandle != NULL ) {
+ CloseHandle(logonHandle);
+ }
+
+ if (INVALID_HANDLE_VALUE != lsaHandle) {
+ UnregisterWithLsa(lsaHandle);
+ }
+
+ return err;
+}
+
+//----------------------------------------------------------------------------
+// Function: IsTaskAlive
//
// Description:
// Checks if a task is alive via a jobobject. Outputs the
@@ -196,7 +782,7 @@ DWORD createTask(__in PCWSTR jobObjName,__in PWSTR cmdLine)
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
-DWORD isTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob)
+DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob)
{
PJOBOBJECT_BASIC_PROCESS_ID_LIST procList;
HANDLE jobObject = NULL;
@@ -246,24 +832,21 @@ DWORD isTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob)
return ERROR_SUCCESS;
}
-//----------------------------------------------------------------------------
-// Function: killTask
-//
-// Description:
-// Kills a task via a jobobject. Outputs the
+//-----------------------------------------------------------------------------
+// Function: KillTask-//-// Description:-// Kills a task via a jobobject. Outputs the
// appropriate information to stdout on success, or stderr on failure.
//
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
-DWORD killTask(PCWSTR jobObjName)
+DWORD KillTask(PCWSTR jobObjName)
{
HANDLE jobObject = OpenJobObject(JOB_OBJECT_TERMINATE, FALSE, jobObjName);
if(jobObject == NULL)
{
DWORD err = GetLastError();
if(err == ERROR_FILE_NOT_FOUND)
- {
+ {
// job object does not exist. assume its not alive
return ERROR_SUCCESS;
}
@@ -275,12 +858,11 @@ DWORD killTask(PCWSTR jobObjName)
return GetLastError();
}
CloseHandle(jobObject);
-
return ERROR_SUCCESS;
}
//----------------------------------------------------------------------------
-// Function: printTaskProcessList
+// Function: PrintTaskProcessList
//
// Description:
// Prints resource usage of all processes in the task jobobject
@@ -288,7 +870,7 @@ DWORD killTask(PCWSTR jobObjName)
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
-DWORD printTaskProcessList(const WCHAR* jobObjName)
+DWORD PrintTaskProcessList(const WCHAR* jobObjName)
{
DWORD i;
PJOBOBJECT_BASIC_PROCESS_ID_LIST procList;
@@ -372,6 +954,21 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[])
{
DWORD dwErrorCode = ERROR_SUCCESS;
TaskCommandOption command = TaskInvalid;
+ wchar_t* cmdLine = NULL;
+ wchar_t buffer[16*1024] = L""; // 32K max command line
+ size_t charCountBufferLeft = sizeof(buffer)/sizeof(wchar_t);
+ int crtArgIndex = 0;
+ size_t argLen = 0;
+ size_t wscatErr = 0;
+ wchar_t* insertHere = NULL;
+
+ enum {
+ ARGC_JOBOBJECTNAME = 2,
+ ARGC_USERNAME,
+ ARGC_PIDFILE,
+ ARGC_COMMAND,
+ ARGC_COMMAND_ARGS
+ };
if (!ParseCommandLine(argc, argv, &command)) {
dwErrorCode = ERROR_INVALID_COMMAND_LINE;
@@ -385,10 +982,57 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[])
{
// Create the task jobobject
//
- dwErrorCode = createTask(argv[2], argv[3]);
+ dwErrorCode = CreateTask(argv[2], argv[3]);
if (dwErrorCode != ERROR_SUCCESS)
{
- ReportErrorCode(L"createTask", dwErrorCode);
+ ReportErrorCode(L"CreateTask", dwErrorCode);
+ goto TaskExit;
+ }
+ } else if (command == TaskCreateAsUser)
+ {
+ // Create the task jobobject as a domain user
+ // createAsUser accepts an open list of arguments. All arguments after the command are
+ // to be passed as argumrnts to the command itself.Here we're concatenating all
+ // arguments after the command into a single arg entry.
+ //
+ cmdLine = argv[ARGC_COMMAND];
+ if (argc > ARGC_COMMAND_ARGS) {
+ crtArgIndex = ARGC_COMMAND;
+ insertHere = buffer;
+ while (crtArgIndex < argc) {
+ argLen = wcslen(argv[crtArgIndex]);
+ wscatErr = wcscat_s(insertHere, charCountBufferLeft, argv[crtArgIndex]);
+ switch (wscatErr) {
+ case 0:
+ // 0 means success;
+ break;
+ case EINVAL:
+ dwErrorCode = ERROR_INVALID_PARAMETER;
+ goto TaskExit;
+ case ERANGE:
+ dwErrorCode = ERROR_INSUFFICIENT_BUFFER;
+ goto TaskExit;
+ default:
+ // This case is not MSDN documented.
+ dwErrorCode = ERROR_GEN_FAILURE;
+ goto TaskExit;
+ }
+ insertHere += argLen;
+ charCountBufferLeft -= argLen;
+ insertHere[0] = L' ';
+ insertHere += 1;
+ charCountBufferLeft -= 1;
+ insertHere[0] = 0;
+ ++crtArgIndex;
+ }
+ cmdLine = buffer;
+ }
+
+ dwErrorCode = CreateTaskAsUser(
+ argv[ARGC_JOBOBJECTNAME], argv[ARGC_USERNAME], argv[ARGC_PIDFILE], cmdLine);
+ if (dwErrorCode != ERROR_SUCCESS)
+ {
+ ReportErrorCode(L"CreateTaskAsUser", dwErrorCode);
goto TaskExit;
}
} else if (command == TaskIsAlive)
@@ -397,10 +1041,10 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[])
//
int isAlive;
int numProcs;
- dwErrorCode = isTaskAlive(argv[2], &isAlive, &numProcs);
+ dwErrorCode = IsTaskAlive(argv[2], &isAlive, &numProcs);
if (dwErrorCode != ERROR_SUCCESS)
{
- ReportErrorCode(L"isTaskAlive", dwErrorCode);
+ ReportErrorCode(L"IsTaskAlive", dwErrorCode);
goto TaskExit;
}
@@ -412,27 +1056,27 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[])
else
{
dwErrorCode = ERROR_TASK_NOT_ALIVE;
- ReportErrorCode(L"isTaskAlive returned false", dwErrorCode);
+ ReportErrorCode(L"IsTaskAlive returned false", dwErrorCode);
goto TaskExit;
}
} else if (command == TaskKill)
{
// Check if task jobobject
//
- dwErrorCode = killTask(argv[2]);
+ dwErrorCode = KillTask(argv[2]);
if (dwErrorCode != ERROR_SUCCESS)
{
- ReportErrorCode(L"killTask", dwErrorCode);
+ ReportErrorCode(L"KillTask", dwErrorCode);
goto TaskExit;
}
} else if (command == TaskProcessList)
{
// Check if task jobobject
//
- dwErrorCode = printTaskProcessList(argv[2]);
+ dwErrorCode = PrintTaskProcessList(argv[2]);
if (dwErrorCode != ERROR_SUCCESS)
{
- ReportErrorCode(L"printTaskProcessList", dwErrorCode);
+ ReportErrorCode(L"PrintTaskProcessList", dwErrorCode);
goto TaskExit;
}
} else
@@ -443,6 +1087,7 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[])
}
TaskExit:
+ ReportErrorCode(L"TaskExit:", dwErrorCode);
return dwErrorCode;
}
@@ -453,10 +1098,12 @@ void TaskUsage()
// ProcessTree.isSetsidSupported()
fwprintf(stdout, L"\
Usage: task create [TASKNAME] [COMMAND_LINE] |\n\
+ task createAsUser [TASKNAME] [USERNAME] [PIDFILE] [COMMAND_LINE] |\n\
task isAlive [TASKNAME] |\n\
task kill [TASKNAME]\n\
task processList [TASKNAME]\n\
Creates a new task jobobject with taskname\n\
+ Creates a new task jobobject with taskname as the user provided\n\
Checks if task jobobject is alive\n\
Kills task jobobject\n\
Prints to stdout a list of processes in the task\n\
diff --git hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc
new file mode 100644
index 0000000..a2e30ad
--- /dev/null
+++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc
@@ -0,0 +1,46 @@
+; // winutils.mc
+
+; // EventLog messages for Hadoop winutils service.
+
+
+LanguageNames=(English=0x409:MSG00409)
+
+
+; // The following are the categories of events.
+
+MessageIdTypedef=WORD
+
+MessageId=0x1
+SymbolicName=SERVICE_CATEGORY
+Language=English
+Service Events
+.
+
+MessageId=0x2
+SymbolicName=LOG_CATEGORY
+Language=English
+Task Events
+.
+
+; // The following are the message definitions.
+
+MessageIdTypedef=DWORD
+
+MessageId=0x80
+SymbolicName=MSG_CHECK_ERROR
+Language=English
+%1. Error %2: %3.
+.
+
+MessageId=0x100
+SymbolicName=MSG_RPC_SERVICE_HAS_STARTED
+Language=English
+The LPC server is listenning.
+.
+
+MessageId=0x200
+SymbolicName=MSG_RPC_SERVICE_HAS_STOPPED
+Language=English
+The LPC server has stopped listenning.
+.
+
diff --git hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
index d4e019e..e2a36be 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
+++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
@@ -1,22 +1,6 @@

Microsoft Visual Studio Solution File, Format Version 11.00
# Visual Studio 2010
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winutils", "winutils.vcxproj", "{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}"
ProjectSection(ProjectDependencies) = postProject
{12131AA7-902E-4A6D-9CE3-043261D22A12} = {12131AA7-902E-4A6D-9CE3-043261D22A12}
@@ -26,26 +10,16 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwinutils", "libwinutils.
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
- Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
- Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
- {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.ActiveCfg = Debug|x64
- {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.Build.0 = Debug|x64
- {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Debug|x64
- {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Debug|x64
- {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.ActiveCfg = Release|Win32
- {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.Build.0 = Release|Win32
+ {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Release|x64
+ {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Release|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.ActiveCfg = Release|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.Build.0 = Release|x64
- {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.ActiveCfg = Debug|x64
- {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.Build.0 = Debug|x64
- {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Debug|x64
- {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Debug|x64
- {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.ActiveCfg = Release|Win32
- {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.Build.0 = Release|Win32
+ {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Release|x64
+ {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.ActiveCfg = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.Build.0 = Release|x64
EndGlobalSection
diff --git hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
index 5b9a195..d736084 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
+++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
@@ -19,18 +19,10 @@
-
- Debug
- Win32
-
Debug
x64
-
- Release
- Win32
-
Release
x64
@@ -42,22 +34,11 @@
winutils
-
- Application
- true
- Unicode
-
Application
true
Unicode
-
- Application
- false
- true
- Unicode
-
Application
false
@@ -67,15 +48,9 @@
-
-
-
-
-
-
@@ -83,74 +58,32 @@
include;$(IncludePath)
-
- true
-
-
- true
-
- ..\..\..\target\winutils\$(Configuration)\
-
-
- false
-
false
- ..\..\..\target\winutils\$(Platform)\$(Configuration)\
- ..\..\..\target\bin\
-
-
-
-
- Level3
- Disabled
- WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
-
-
- Console
- true
-
-
Level4
Disabled
- WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ WIN32;_CONSOLE;_DEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)
Console
true
-
-
- Level3
-
-
- MaxSpeed
- true
- true
- WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
-
-
- Console
- true
- true
- true
-
-
Level3
- MaxSpeed
+
+ Disabled
true
true
- WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+ WIN32;_CONSOLE;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)
Console
@@ -159,7 +92,40 @@
true
+
+
+ $(IntermediateOutputPath)
+
+
+ Compiling Messages
+ mc.exe $(TargetName).mc -z $(TargetName)_msg -r $(IntermediateOutputPath) -h $(IntermediateOutputPath) -U
+ $(IntermediateOutputPath)$(TargetName)_msg.rc;$(IntermediateOutputPath)$(TargetName)_msg.h
+
+
+ true
+ X64
+ $(IntermediateOutputPath)
+ true
+ true
+ true
+ 2
+
+
+
+ Midl
+ ClCompile,ResourceCompile
+
+
+
+
+
+
+
+
+
+
+
@@ -179,4 +145,4 @@
-
\ No newline at end of file
+
diff --git hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java
index 588b217..953039d 100644
--- hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java
+++ hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java
@@ -20,10 +20,12 @@
import static org.junit.Assert.*;
import static org.junit.Assume.assumeTrue;
+import static org.junit.matchers.JUnitMatchers.containsString;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
+import java.io.FileWriter;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
@@ -33,7 +35,7 @@
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assume.*;
+
import static org.hamcrest.CoreMatchers.*;
/**
@@ -521,4 +523,26 @@ public void testReadLink() throws IOException {
assertThat(ece.getExitCode(), is(1));
}
}
+
+ @SuppressWarnings("deprecation")
+ @Test(timeout=10000)
+ public void testTaskCreate() throws IOException {
+ File batch = new File(TEST_DIR, "testTaskCreate.cmd");
+ File proof = new File(TEST_DIR, "testTaskCreate.out");
+ FileWriter fw = new FileWriter(batch);
+ String testNumber = String.format("%f", Math.random());
+ fw.write(String.format("echo %s > \"%s\"", testNumber, proof.getAbsolutePath()));
+ fw.close();
+
+ assertFalse(proof.exists());
+
+ Shell.execCommand(Shell.WINUTILS, "task", "create", "testTaskCreate" + testNumber,
+ batch.getAbsolutePath());
+
+ assertTrue(proof.exists());
+
+ String outNumber = FileUtils.readFileToString(proof);
+
+ assertThat(outNumber, containsString(testNumber));
+ }
}
diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java
index 2f8b84d..1e2d16e 100644
--- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java
+++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java
@@ -296,7 +296,7 @@ public static boolean isAlive(String pid) {
return false;
} catch (IOException ioe) {
LOG.warn("Error executing shell command "
- + Arrays.toString(shexec.getExecString()) + ioe);
+ + shexec.toString() + ioe);
return false;
}
return (shexec.getExitCode() == 0 ? true : false);
@@ -321,7 +321,7 @@ public static boolean isProcessGroupAlive(String pgrpId) {
return false;
} catch (IOException ioe) {
LOG.warn("Error executing shell command "
- + Arrays.toString(shexec.getExecString()) + ioe);
+ + shexec.toString() + ioe);
return false;
}
return (shexec.getExitCode() == 0 ? true : false);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 034ec4f..dcb57a6 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -894,7 +894,13 @@
public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT =
1000;
-
+
+ /**
+ /* The Windows group that the windows-secure-container-executor should run as.
+ */
+ public static final String NM_WINDOWS_SECURE_CONTAINER_GROUP =
+ NM_PREFIX + "windows-secure-container-executor.group";
+
/** T-file compression types used to compress aggregated logs.*/
public static final String NM_LOG_AGG_COMPRESSION_TYPE =
NM_PREFIX + "log-aggregation.compression-type";
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 7391872..eed2175 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -79,6 +79,25 @@ public Configuration getConf() {
public abstract void init() throws IOException;
/**
+ * On Windows the ContainerLaunch creates a temporary empty jar to workaround the CLASSPATH length
+ * In a secure cluster this jar must be localized so that the container has access to it
+ * This function localizes on-demand the jar.
+ *
+ * @param classPathJar
+ * @param owner
+ * @throws IOException
+ */
+ public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner) throws IOException {
+ // None secure executor simply use the classpath create din the NM fprivate folder
+ return classPathJar;
+ }
+
+
+ public Path getContainerClasspathJarPrivateDir(String pwd) throws IOException {
+ return new Path(pwd);
+ }
+
+ /**
* Prepare the environment for containers in this application to execute.
* For $x in local.dirs
* create $x/$user/$appId
@@ -98,7 +117,7 @@ public Configuration getConf() {
*/
public abstract void startLocalizer(Path nmPrivateContainerTokens,
InetSocketAddress nmAddr, String user, String appId, String locId,
- List localDirs, List logDirs)
+ LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException;
@@ -118,8 +137,8 @@ public abstract void startLocalizer(Path nmPrivateContainerTokens,
*/
public abstract int launchContainer(Container container,
Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath,
- String user, String appId, Path containerWorkDir, List localDirs,
- List logDirs) throws IOException;
+ String user, String appId, Path containerWorkDir,
+ List localDirs, List logDirs) throws IOException;
public abstract boolean signalContainer(String user, String pid,
Signal signal)
@@ -264,8 +283,8 @@ protected Path getPidFilePath(ContainerId containerId) {
* and associate the given groupId in a process group. On
* non-Windows, groupId is ignored.
*/
- protected static String[] getRunCommand(String command, String groupId,
- Configuration conf) {
+ protected String[] getRunCommand(String command, String groupId,
+ String userName, Path pidFile, Configuration conf) {
boolean containerSchedPriorityIsSet = false;
int containerSchedPriorityAdjustment =
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;
@@ -396,5 +415,4 @@ public void run() {
}
}
}
-
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index a7af1c5..e29244c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import com.google.common.base.Optional;
+
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
@@ -31,9 +32,11 @@
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -41,6 +44,7 @@
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
+import org.apache.hadoop.util.Shell.ICommandExecutor;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -61,7 +65,7 @@
private static final int WIN_MAX_PATH = 260;
- private final FileContext lfs;
+ protected final FileContext lfs;
public DefaultContainerExecutor() {
try {
@@ -75,32 +79,43 @@ public DefaultContainerExecutor() {
this.lfs = lfs;
}
+ protected void copyFile(Path src, Path dst, String owner) throws IOException {
+ lfs.util().copy(src, dst);
+ }
+
+ protected void setScriptExecutable(Path script, String owner) throws IOException {
+ lfs.setPermission(script, ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);
+ }
+
@Override
public void init() throws IOException {
// nothing to do or verify here
}
-
+
@Override
public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
- List localDirs, List logDirs)
+ LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
+ List localDirs = dirsHandler.getLocalDirs();
+ List logDirs = dirsHandler.getLogDirs();
+
ContainerLocalizer localizer =
new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
RecordFactoryProvider.getRecordFactory(getConf()));
-
+
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId);
- createAppLogDirs(appId, logDirs);
+ createAppLogDirs(appId, logDirs, user);
// TODO: Why pick first app dir. The same in LCE why not random?
Path appStorageDir = getFirstApplicationDir(localDirs, user, appId);
String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
Path tokenDst = new Path(appStorageDir, tokenFn);
- lfs.util().copy(nmPrivateContainerTokensPath, tokenDst);
+ copyFile(nmPrivateContainerTokensPath, tokenDst, user);
LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
lfs.setWorkingDirectory(appStorageDir);
LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
@@ -113,7 +128,7 @@ public int launchContainer(Container container,
Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath,
String userName, String appId, Path containerWorkDir,
List localDirs, List logDirs) throws IOException {
-
+
FsPermission dirPerm = new FsPermission(APPDIR_PERM);
ContainerId containerId = container.getContainerId();
@@ -129,30 +144,30 @@ public int launchContainer(Container container,
Path appCacheDir = new Path(userdir, ContainerLocalizer.APPCACHE);
Path appDir = new Path(appCacheDir, appIdStr);
Path containerDir = new Path(appDir, containerIdStr);
- createDir(containerDir, dirPerm, true);
+ createDir(containerDir, dirPerm, true, userName);
}
// Create the container log-dirs on all disks
- createContainerLogDirs(appIdStr, containerIdStr, logDirs);
+ createContainerLogDirs(appIdStr, containerIdStr, logDirs, userName);
Path tmpDir = new Path(containerWorkDir,
YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
- createDir(tmpDir, dirPerm, false);
+ createDir(tmpDir, dirPerm, false, userName);
- // copy launch script to work dir
- Path launchDst =
- new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
- lfs.util().copy(nmPrivateContainerScriptPath, launchDst);
// copy container tokens to work dir
Path tokenDst =
new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE);
- lfs.util().copy(nmPrivateTokensPath, tokenDst);
+ copyFile(nmPrivateTokensPath, tokenDst, userName);
+
+ // copy launch script to work dir
+ Path launchDst =
+ new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
+ copyFile(nmPrivateContainerScriptPath, launchDst, userName);
// Create new local launch wrapper script
- LocalWrapperScriptBuilder sb = Shell.WINDOWS ?
- new WindowsLocalWrapperScriptBuilder(containerIdStr, containerWorkDir) :
- new UnixLocalWrapperScriptBuilder(containerWorkDir);
+ LocalWrapperScriptBuilder sb = getLocalWrapperScriptBuilder(
+ containerIdStr, containerWorkDir);
// Fail fast if attempting to launch the wrapper script would fail due to
// Windows path length limitation.
@@ -173,25 +188,19 @@ public int launchContainer(Container container,
+ " was marked as inactive. Returning terminated error");
return ExitCode.TERMINATED.getExitCode();
}
-
+
// create log dir under app
// fork script
- ShellCommandExecutor shExec = null;
+ Shell.ICommandExecutor shExec = null;
try {
- lfs.setPermission(launchDst,
- ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);
- lfs.setPermission(sb.getWrapperScriptPath(),
- ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);
+ setScriptExecutable(launchDst, userName);
+ setScriptExecutable(sb.getWrapperScriptPath(), userName);
- // Setup command to run
- String[] command = getRunCommand(sb.getWrapperScriptPath().toString(),
- containerIdStr, this.getConf());
-
- LOG.info("launchContainer: " + Arrays.toString(command));
- shExec = new ShellCommandExecutor(
- command,
+ shExec = buildCommandExecutor(sb.getWrapperScriptPath().toString(),
+ containerIdStr, userName, pidFile, this.getConf(),
new File(containerWorkDir.toUri().getPath()),
- container.getLaunchContext().getEnvironment()); // sanitized env
+ container.getLaunchContext().getEnvironment());
+
if (isContainerActive(containerId)) {
shExec.execute();
}
@@ -236,12 +245,33 @@ public int launchContainer(Container container,
}
return exitCode;
} finally {
- ; //
+ if (null != shExec) shExec.dispose();
}
return 0;
}
- private abstract class LocalWrapperScriptBuilder {
+ protected ICommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr,
+ String userName, Path pidFile, Configuration conf, File wordDir, Map environment)
+ throws IOException {
+
+ String[] command = getRunCommand(wrapperScriptPath,
+ containerIdStr, userName, pidFile, this.getConf());
+
+ LOG.info("launchContainer: " + Arrays.toString(command));
+ return new ShellCommandExecutor(
+ command,
+ wordDir,
+ environment);
+ }
+
+ protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder(
+ String containerIdStr, Path containerWorkDir) {
+ return Shell.WINDOWS ?
+ new WindowsLocalWrapperScriptBuilder(containerIdStr, containerWorkDir) :
+ new UnixLocalWrapperScriptBuilder(containerWorkDir);
+ }
+
+ protected abstract class LocalWrapperScriptBuilder {
private final Path wrapperScriptPath;
@@ -408,7 +438,7 @@ public static boolean containerIsAlive(String pid) throws IOException {
* @param signal signal to send
* (for logging).
*/
- private void killContainer(String pid, Signal signal) throws IOException {
+ protected void killContainer(String pid, Signal signal) throws IOException {
new ShellCommandExecutor(Shell.getSignalKillCommand(signal.getValue(), pid))
.execute();
}
@@ -449,7 +479,7 @@ public void deleteAsUser(String user, Path subDir, Path... baseDirs)
* $logdir/$user/$appId */
static final short LOGDIR_PERM = (short)0710;
- private Path getFirstApplicationDir(List localDirs, String user,
+ protected Path getFirstApplicationDir(List localDirs, String user,
String appId) {
return getApplicationDir(new Path(localDirs.get(0)), user, appId);
}
@@ -472,8 +502,8 @@ private Path getFileCacheDir(Path base, String user) {
ContainerLocalizer.FILECACHE);
}
- private void createDir(Path dirPath, FsPermission perms,
- boolean createParent) throws IOException {
+ protected void createDir(Path dirPath, FsPermission perms,
+ boolean createParent, String user) throws IOException {
lfs.mkdir(dirPath, perms, createParent);
if (!perms.equals(perms.applyUMask(lfs.getUMask()))) {
lfs.setPermission(dirPath, perms);
@@ -493,7 +523,7 @@ void createUserLocalDirs(List localDirs, String user)
for (String localDir : localDirs) {
// create $local.dir/usercache/$user and its immediate parent
try {
- createDir(getUserCacheDir(new Path(localDir), user), userperms, true);
+ createDir(getUserCacheDir(new Path(localDir), user), userperms, true, user);
} catch (IOException e) {
LOG.warn("Unable to create the user directory : " + localDir, e);
continue;
@@ -529,7 +559,7 @@ void createUserCacheDirs(List localDirs, String user)
Path localDirPath = new Path(localDir);
final Path appDir = getAppcacheDir(localDirPath, user);
try {
- createDir(appDir, appCachePerms, true);
+ createDir(appDir, appCachePerms, true, user);
appcacheDirStatus = true;
} catch (IOException e) {
LOG.warn("Unable to create app cache directory : " + appDir, e);
@@ -537,7 +567,7 @@ void createUserCacheDirs(List localDirs, String user)
// create $local.dir/usercache/$user/filecache
final Path distDir = getFileCacheDir(localDirPath, user);
try {
- createDir(distDir, fileperms, true);
+ createDir(distDir, fileperms, true, user);
distributedCacheDirStatus = true;
} catch (IOException e) {
LOG.warn("Unable to create file cache directory : " + distDir, e);
@@ -570,7 +600,7 @@ void createAppDirs(List localDirs, String user, String appId)
Path fullAppDir = getApplicationDir(new Path(localDir), user, appId);
// create $local.dir/usercache/$user/appcache/$appId
try {
- createDir(fullAppDir, appperms, true);
+ createDir(fullAppDir, appperms, true, user);
initAppDirStatus = true;
} catch (IOException e) {
LOG.warn("Unable to create app directory " + fullAppDir.toString(), e);
@@ -586,7 +616,7 @@ void createAppDirs(List localDirs, String user, String appId)
/**
* Create application log directories on all disks.
*/
- void createAppLogDirs(String appId, List logDirs)
+ void createAppLogDirs(String appId, List logDirs, String user)
throws IOException {
boolean appLogDirStatus = false;
@@ -595,7 +625,7 @@ void createAppLogDirs(String appId, List logDirs)
// create $log.dir/$appid
Path appLogDir = new Path(rootLogDir, appId);
try {
- createDir(appLogDir, appLogDirPerms, true);
+ createDir(appLogDir, appLogDirPerms, true, user);
} catch (IOException e) {
LOG.warn("Unable to create the app-log directory : " + appLogDir, e);
continue;
@@ -612,7 +642,7 @@ void createAppLogDirs(String appId, List logDirs)
* Create application log directories on all disks.
*/
void createContainerLogDirs(String appId, String containerId,
- List logDirs) throws IOException {
+ List logDirs, String user) throws IOException {
boolean containerLogDirStatus = false;
FsPermission containerLogDirPerms = new FsPermission(LOGDIR_PERM);
@@ -621,7 +651,7 @@ void createContainerLogDirs(String appId, String containerId,
Path appLogDir = new Path(rootLogDir, appId);
Path containerLogDir = new Path(appLogDir, containerId);
try {
- createDir(containerLogDir, containerLogDirPerms, true);
+ createDir(containerLogDir, containerLogDirPerms, true, user);
} catch (IOException e) {
LOG.warn("Unable to create the container-log directory : "
+ appLogDir, e);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 804864e..69c8db3 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import com.google.common.base.Optional;
+
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -194,9 +195,12 @@ public void init() throws IOException {
@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
- List localDirs, List logDirs)
+ LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
+ List localDirs = dirsHandler.getLocalDirs();
+ List logDirs = dirsHandler.getLogDirs();
+
verifyUsernamePattern(user);
String runAsUser = getRunAsUser(user);
List command = new ArrayList();
@@ -219,15 +223,7 @@ public void startLocalizer(Path nmPrivateContainerTokensPath,
if (javaLibPath != null) {
command.add("-Djava.library.path=" + javaLibPath);
}
- command.add(ContainerLocalizer.class.getName());
- command.add(user);
- command.add(appId);
- command.add(locId);
- command.add(nmAddr.getHostName());
- command.add(Integer.toString(nmAddr.getPort()));
- for (String dir : localDirs) {
- command.add(dir);
- }
+ ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, localDirs);
String[] commandArray = command.toArray(new String[command.size()]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
if (LOG.isDebugEnabled()) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
new file mode 100644
index 0000000..30018ea
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -0,0 +1,440 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.io.nativeio.NativeIO.WinutilsProcessStub;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.Shell.ICommandExecutor;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+
+/**
+ * Windows secure container executor (WSCE).
+ * This class offers a secure container executor on Windows, similar to the LinuxContainerExecutor
+ * As the NM does not run on a high privileged context, this class delegates elevated operations
+ * to the helper hadoopwintuilsvc, implemented by the winutils.exe running as a service.
+ * JNI and LRPC is used to communicate with the privileged service.
+ */
+public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
+
+ private static final Log LOG = LogFactory
+ .getLog(WindowsSecureContainerExecutor.class);
+
+ public static final String LOCALIZER_PID_FORMAT = "STAR_LOCALIZER_%s";
+
+ /**
+ * A shell script wrapper builder for WSCE.
+ * Overwrites the default behavior to remove the creation of the PID file in the script wrapper.
+ * WSCE creates the pid file as part of launching the task in winutils
+ */
+ private class WindowsSecureWrapperScriptBuilder
+ extends LocalWrapperScriptBuilder {
+
+ public WindowsSecureWrapperScriptBuilder(Path containerWorkDir) {
+ super(containerWorkDir);
+ }
+
+ @Override
+ protected void writeLocalWrapperScript(Path launchDst, Path pidFile, PrintStream pout) {
+ pout.format("@call \"%s\"", launchDst);
+ }
+ }
+
+ /**
+ * This is a skeleton file system used to elevate certain operations.
+ * WSCE has to create container dirs under local/userchache/$user but
+ * this dir itself is owned by $user, with chmod 750. As ther NM has no
+ * write access, it must delegate the write operations to the privileged
+ * hadoopwintuilsvc.
+ */
+ private static class ElevatedFileSystem extends DelegateToFileSystem {
+
+ /**
+ * This overwrites certain RawLocalSystem operations to be performed by a privileged process.
+ *
+ */
+ private static class ElevatedRawLocalFilesystem extends RawLocalFileSystem {
+
+ @Override
+ protected boolean mkOneDir(File p2f) throws IOException {
+ Path path = new Path(p2f.getAbsolutePath());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:mkOneDir: %s", path));
+ }
+ boolean ret = false;
+
+ // File.mkdir returns false, does not throw. Must mimic it.
+ try {
+ NativeIO.Elevated.mkdir(path);
+ ret = true;
+ }
+ catch(Throwable e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:mkOneDir: %s",
+ org.apache.hadoop.util.StringUtils.stringifyException(e)));
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public void setPermission(Path p, FsPermission permission) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:setPermission: %s %s", p, permission));
+ }
+ //super.setPermission(p, permission);
+ NativeIO.Elevated.chmod(p, permission.toShort());
+ }
+
+ @Override
+ public void setOwner(Path p, String username, String groupname) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:setOwner: %s %s %s", p, username, groupname));
+ }
+ NativeIO.Elevated.chown(p, username, groupname);
+ }
+
+ @Override
+ protected OutputStream createOutputStream(Path f, boolean append) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:create: %s %b", f, append));
+ }
+ return NativeIO.Elevated.create(f, append);
+ }
+
+ @Override
+ public boolean delete(Path p, boolean recursive) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:delete: %s %b", p, recursive));
+ }
+ return super.delete(p, recursive);
+ }
+ }
+
+ protected ElevatedFileSystem() throws IOException, URISyntaxException {
+ super(FsConstants.LOCAL_FS_URI,
+ new ElevatedRawLocalFilesystem(),
+ new Configuration(),
+ FsConstants.LOCAL_FS_URI.getScheme(),
+ false);
+ }
+ }
+
+ private static class WintuilsProcessStubExecutor implements Shell.ICommandExecutor {
+ private WinutilsProcessStub processStub;
+ private StringBuilder output = new StringBuilder();
+ private int exitCode;
+
+ private enum State {
+ INIT,
+ RUNNING,
+ COMPLETE
+ };
+
+ private State state;;
+
+ private final String cwd;
+ private final String jobName;
+ private final String userName;
+ private final String pidFile;
+ private final String cmdLine;
+
+ public WintuilsProcessStubExecutor(
+ String cwd,
+ String jobName,
+ String userName,
+ String pidFile,
+ String cmdLine) {
+ this.cwd = cwd;
+ this.jobName = jobName;
+ this.userName = userName;
+ this.pidFile = pidFile;
+ this.cmdLine = cmdLine;
+ this.state = State.INIT;
+ }
+
+ private void assertComplete() throws IOException {
+ if (state != State.COMPLETE) {
+ throw new IOException("Process is not complete");
+ }
+ }
+
+ public String getOutput () throws IOException {
+ assertComplete();
+ return output.toString();
+ }
+
+ public int getExitCode() throws IOException {
+ assertComplete();
+ return exitCode;
+ }
+
+ public void validateResult() throws IOException {
+ assertComplete();
+ if (0 != exitCode) {
+ LOG.warn(output.toString());
+ throw new IOException("Processs exit code is:" + exitCode);
+ }
+ }
+
+ private Thread startStreamReader(final InputStream stream) throws IOException {
+ Thread streamReaderThread = new Thread() {
+
+ @Override
+ public void run() {
+ try
+ {
+ BufferedReader rdr = new BufferedReader(
+ new InputStreamReader(stream));
+ String line = rdr.readLine();
+ while((line != null) && !isInterrupted()) {
+ synchronized(output) {
+ output.append(line);
+ output.append(System.getProperty("line.separator"));
+ }
+ line = rdr.readLine();
+ }
+ }
+ catch(Throwable t) {
+ LOG.error("Error occured reading the process stdout", t);
+ }
+ }
+ };
+ streamReaderThread.start();
+ return streamReaderThread;
+ }
+
+ public void execute() throws IOException {
+ if (state != State.INIT) {
+ throw new IOException("Process is already started");
+ }
+ processStub = NativeIO.createTaskAsUser(cwd, jobName, userName, pidFile, cmdLine);
+ state = State.RUNNING;
+
+ Thread stdOutReader = startStreamReader(processStub.getInputStream());
+ Thread stdErrReader = startStreamReader(processStub.getErrorStream());
+
+ try {
+ processStub.resume();
+ processStub.waitFor();
+ stdOutReader.join();
+ stdErrReader.join();
+ }
+ catch(InterruptedException ie) {
+ throw new IOException(ie);
+ }
+
+ exitCode = processStub.exitValue();
+ state = State.COMPLETE;
+ }
+
+ @Override
+ public void dispose() {
+ if (processStub != null) {
+ processStub.dispose();
+ }
+ }
+ }
+
+ private String nodeManagerGroup;
+
+ public WindowsSecureContainerExecutor() throws IOException, URISyntaxException {
+ super(FileContext.getFileContext(new ElevatedFileSystem(), new Configuration()));
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ nodeManagerGroup = conf.get(YarnConfiguration.NM_WINDOWS_SECURE_CONTAINER_GROUP);
+ }
+
+ @Override
+ protected String[] getRunCommand(String command, String groupId,
+ String userName, Path pidFile, Configuration conf) {
+ File f = new File(command);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("getRunCommand: %s exists:%b", command, f.exists()));
+ }
+ return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId, userName,
+ pidFile.toString(), "cmd /c " + command };
+ }
+
+ @Override
+ protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder(
+ String containerIdStr, Path containerWorkDir) {
+ return new WindowsSecureWrapperScriptBuilder(containerWorkDir);
+ }
+
+ @Override
+ protected void copyFile(Path src, Path dst, String owner) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("copyFile: %s -> %s owner:%s", src.toString(), dst.toString(), owner));
+ }
+ NativeIO.Elevated.copy(src, dst, true);
+ NativeIO.Elevated.chown(dst, owner, nodeManagerGroup);
+ }
+
+ @Override
+ protected void createDir(Path dirPath, FsPermission perms,
+ boolean createParent, String owner) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("createDir: %s perm:%s owner:%s", dirPath.toString(), perms.toString(), owner));
+ }
+
+ super.createDir(dirPath, perms, createParent, owner);
+ lfs.setOwner(dirPath, owner, nodeManagerGroup);
+ }
+
+ @Override
+ protected void setScriptExecutable(Path script, String owner) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("setScriptExecutable: %s owner:%s", script.toString(), owner));
+ }
+ super.setScriptExecutable(script, owner);
+ NativeIO.Elevated.chown(script, owner, nodeManagerGroup);
+ }
+
+ @Override
+ public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("localizeClasspathJar: %s %s o:%s", classPathJar, pwd, owner));
+ }
+ createDir(pwd, new FsPermission(APPDIR_PERM), true, owner);
+ String fileName = classPathJar.getName();
+ Path dst = new Path(pwd, fileName);
+ NativeIO.Elevated.move(classPathJar, dst, true);
+ NativeIO.Elevated.chown(dst, owner, nodeManagerGroup);
+ return dst;
+ }
+
+ @Override
+ public void startLocalizer(Path nmPrivateContainerTokens,
+ InetSocketAddress nmAddr, String user, String appId, String locId,
+ LocalDirsHandlerService dirsHandler) throws IOException,
+ InterruptedException {
+
+ List localDirs = dirsHandler.getLocalDirs();
+ List logDirs = dirsHandler.getLogDirs();
+
+ Path classpathJarPrivateDir = dirsHandler.getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR);
+ createUserLocalDirs(localDirs, user);
+ createUserCacheDirs(localDirs, user);
+ createAppDirs(localDirs, user, appId);
+ createAppLogDirs(appId, logDirs, user);
+
+
+ // TODO: Why pick first app dir. The same in LCE why not random?
+ Path appStorageDir = getFirstApplicationDir(localDirs, user, appId);
+
+ String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
+ Path tokenDst = new Path(appStorageDir, tokenFn);
+ copyFile(nmPrivateContainerTokens, tokenDst, user);
+
+ File cwdApp = new File(appStorageDir.toString());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("cwdApp: %s", cwdApp));
+ }
+
+ List command ;
+
+ command = new ArrayList();
+
+ //use same jvm as parent
+ File jvm = new File(new File(System.getProperty("java.home"), "bin"), "java.exe");
+ command.add(jvm.toString());
+
+
+ // Build a temp classpath jar. See ContainerLaunch.sanitizeEnv().
+ // Passing CLASSPATH explicitly is *way* too long for command line.
+ String classPath = System.getProperty("java.class.path");
+ Map env = new HashMap(System.getenv());
+ String classPathJar = FileUtil.createJarWithClassPath(classPath, classpathJarPrivateDir, env);
+ classPathJar = localizeClasspathJar(
+ new Path(classPathJar), new Path(cwdApp.getPath()), user).toString();
+ command.add("-classpath");
+ command.add(classPathJar);
+
+ String javaLibPath = System.getProperty("java.library.path");
+ if (javaLibPath != null) {
+ command.add("-Djava.library.path=" + javaLibPath);
+ }
+
+ ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, localDirs);
+
+ String cmdLine = StringUtils.join(command, " ");
+
+ String localizerPid = String.format(LOCALIZER_PID_FORMAT, locId);
+
+ WintuilsProcessStubExecutor stubExecutor = new WintuilsProcessStubExecutor(
+ cwdApp.getAbsolutePath(),
+ localizerPid, user, "nul:", cmdLine);
+ try {
+ stubExecutor.execute();
+ stubExecutor.validateResult();
+ }
+ finally {
+ stubExecutor.dispose();
+ try
+ {
+ killContainer(localizerPid, Signal.KILL);
+ }
+ catch(Throwable e) {
+ LOG.warn(String.format("An exception occured during the cleanup of localizer job %s:\n%s",
+ localizerPid, org.apache.hadoop.util.StringUtils.stringifyException(e)));
+ }
+ }
+ }
+
+ @Override
+ protected ICommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr,
+ String userName, Path pidFile, Configuration conf, File wordDir, Map environment)
+ throws IOException {
+
+ return new WintuilsProcessStubExecutor(
+ wordDir.toString(),
+ containerIdStr, userName, pidFile.toString(), "cmd /c " + wrapperScriptPath);
+ }
+}
+
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index cee6a40..ce97dbc 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -212,7 +212,9 @@ public Integer call() {
+ Path.SEPARATOR
+ String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
containerIdStr));
-
+ Path nmPrivateClasspathJarDir =
+ dirsHandler.getLocalPathForWrite(
+ getContainerPrivateDir(appIdStr, containerIdStr));
DataOutputStream containerScriptOutStream = null;
DataOutputStream tokensOutStream = null;
@@ -263,7 +265,7 @@ public Integer call() {
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
- localResources);
+ localResources, nmPrivateClasspathJarDir);
// Write out the environment
writeLaunchEnv(containerScriptOutStream, environment, localResources,
@@ -658,7 +660,8 @@ private static void putEnvIfAbsent(
public void sanitizeEnv(Map environment, Path pwd,
List appDirs, List containerLogDirs,
- Map> resources) throws IOException {
+ Map> resources,
+ Path nmPrivateClasspathJarDir) throws IOException {
/**
* Non-modifiable environment variables
*/
@@ -722,6 +725,7 @@ public void sanitizeEnv(Map environment, Path pwd,
// TODO: Remove Windows check and use this approach on all platforms after
// additional testing. See YARN-358.
if (Shell.WINDOWS) {
+
String inputClassPath = environment.get(Environment.CLASSPATH.name());
if (inputClassPath != null && !inputClassPath.isEmpty()) {
StringBuilder newClassPath = new StringBuilder(inputClassPath);
@@ -765,8 +769,10 @@ public void sanitizeEnv(Map environment, Path pwd,
mergedEnv.putAll(environment);
String classPathJar = FileUtil.createJarWithClassPath(
- newClassPath.toString(), pwd, mergedEnv);
- environment.put(Environment.CLASSPATH.name(), classPathJar);
+ newClassPath.toString(), nmPrivateClasspathJarDir, mergedEnv);
+ // In a secure cluster the classpath jar must be localized to grant access
+ Path localizedClassPathJar = exec.localizeClasspathJar(new Path(classPathJar), pwd, container.getUser());
+ environment.put(Environment.CLASSPATH.name(), localizedClassPathJar.toString());
}
}
// put AuxiliaryService data to environment
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index f05f49c..762565b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -312,6 +312,31 @@ private LocalizerStatus createStatus() throws InterruptedException {
status.addAllResources(currentResources);
return status;
}
+
+ /**
+ * Adds the ContainerLocalizer arguments for a @{link ShellCommandExecutor},
+ * as expected by ContainerLocalizer.main
+ * @param command the current ShellCommandExecutor command line
+ * @param user localization user
+ * @param appId localized app id
+ * @param locId localizer id
+ * @param nmAddr nodemanager address
+ * @param localDirs list of local dirs
+ */
+ public static void buildMainArgs(List command,
+ String user, String appId, String locId,
+ InetSocketAddress nmAddr, List localDirs) {
+
+ command.add(ContainerLocalizer.class.getName());
+ command.add(user);
+ command.add(appId);
+ command.add(locId);
+ command.add(nmAddr.getHostName());
+ command.add(Integer.toString(nmAddr.getPort()));
+ for(String dir : localDirs) {
+ command.add(dir);
+ }
+ }
public static void main(String[] argv) throws Throwable {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
@@ -344,10 +369,13 @@ public static void main(String[] argv) throws Throwable {
new ContainerLocalizer(FileContext.getLocalFSFileContext(), user,
appId, locId, localDirs,
RecordFactoryProvider.getRecordFactory(null));
- System.exit(localizer.runLocalization(nmAddr));
+ int nRet = localizer.runLocalization(nmAddr);
+ LOG.info(String.format("nRet: %d", nRet));
+ System.exit(nRet);
} catch (Throwable e) {
// Print error to stdout so that LCE can use it.
e.printStackTrace(System.out);
+ LOG.error("Exception in main:", e);
throw e;
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 64a0b37..36b4578 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1071,15 +1071,14 @@ public void run() {
// 1) write credentials to private dir
writeCredentials(nmPrivateCTokensPath);
// 2) exec initApplication and wait
- List localDirs = dirsHandler.getLocalDirs();
- List logDirs = dirsHandler.getLogDirs();
if (dirsHandler.areDisksHealthy()) {
exec.startLocalizer(nmPrivateCTokensPath, localizationServerAddress,
context.getUser(),
ConverterUtils.toString(
context.getContainerId().
getApplicationAttemptId().getApplicationId()),
- localizerId, localDirs, logDirs);
+ localizerId,
+ dirsHandler);
} else {
throw new IOException("All disks failed. "
+ dirsHandler.getDisksHealthReport());
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java
index c04ec29..fd3634b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerExecutor.java
@@ -27,11 +27,13 @@
import static org.junit.Assert.*;
public class TestContainerExecutor {
+
+ private ContainerExecutor containerExecutor = new DefaultContainerExecutor();
@Test (timeout = 5000)
public void testRunCommandNoPriority() throws Exception {
Configuration conf = new Configuration();
- String[] command = ContainerExecutor.getRunCommand("echo", "group1", conf);
+ String[] command = containerExecutor.getRunCommand("echo", "group1", "user", null, conf);
assertTrue("first command should be the run command for the platform",
command[0].equals(Shell.WINUTILS) || command[0].equals("bash"));
}
@@ -40,7 +42,7 @@ public void testRunCommandNoPriority() throws Exception {
public void testRunCommandwithPriority() throws Exception {
Configuration conf = new Configuration();
conf.setInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 2);
- String[] command = ContainerExecutor.getRunCommand("echo", "group1", conf);
+ String[] command = containerExecutor.getRunCommand("echo", "group1", "user", null, conf);
if (Shell.WINDOWS) {
// windows doesn't currently support
assertEquals("first command should be the run command for the platform",
@@ -54,7 +56,7 @@ public void testRunCommandwithPriority() throws Exception {
// test with negative number
conf.setInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, -5);
- command = ContainerExecutor.getRunCommand("echo", "group1", conf);
+ command = containerExecutor.getRunCommand("echo", "group1", "user", null, conf);
if (Shell.WINDOWS) {
// windows doesn't currently support
assertEquals("first command should be the run command for the platform",
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
index 9c86c71..f6f0e9f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
@@ -199,7 +199,7 @@ public void testDirPermissions() throws Exception {
Assert.assertEquals(appDirPerm, stats.getPermission());
}
- executor.createAppLogDirs(appId, logDirs);
+ executor.createAppLogDirs(appId, logDirs, user);
for (String dir : logDirs) {
FileStatus stats = lfs.getFileStatus(new Path(dir, appId));
@@ -277,7 +277,7 @@ public Object answer(InvocationOnMock invocationOnMock)
mockExec.createUserLocalDirs(localDirs, appSubmitter);
mockExec.createUserCacheDirs(localDirs, appSubmitter);
mockExec.createAppDirs(localDirs, appSubmitter, appId);
- mockExec.createAppLogDirs(appId, logDirs);
+ mockExec.createAppLogDirs(appId, logDirs, appSubmitter);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 2e9e8b1..d54367a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -185,7 +185,7 @@ public void testStartLocalizer() throws IOException {
Path nmPrivateCTokensPath= new Path("file:///bin/nmPrivateCTokensPath");
try {
- mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler.getLocalDirs(), dirsHandler.getLogDirs());
+ mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler);
List result=readMockParams();
Assert.assertEquals(result.size(), 17);
Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index ed59ddd..187c20b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -737,7 +737,7 @@ public boolean matches(Object o) {
ArgumentCaptor tokenPathCaptor = ArgumentCaptor.forClass(Path.class);
verify(exec).startLocalizer(tokenPathCaptor.capture(),
isA(InetSocketAddress.class), eq("user0"), eq(appStr), eq(ctnrStr),
- isA(List.class), isA(List.class));
+ isA(LocalDirsHandlerService.class));
Path localizationTokenPath = tokenPathCaptor.getValue();
// heartbeat from localizer
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm
new file mode 100644
index 0000000..50ff345
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm
@@ -0,0 +1,148 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+ ---
+ YARN Secure Containers
+ ---
+ ---
+ ${maven.build.timestamp}
+
+YARN Secure Containers
+
+%{toc|section=1|fromDepth=0|toDepth=3}
+
+* {Overview}
+
+ YARN containers in a secure cluster use the operating system facilities to offer
+ execution isolation for containers. Secure containers execute under the credentials
+ of the job user. The operating system enforces access restriction for the container.
+ The container must run as the use that submitted the application.
+
+ Secure Containers work only in the context of secured YARN clusters.
+
+ ** Container isolation requirements
+
+ The container executor must access the local files and directories needed by the
+ container such as jars, configuration files, log files, shared objects etc. Although
+ it is launched by the NodeManager, the container should not have access to the
+ NodeManager private files and configuration. Container running applications
+ submitted by different users should be isolated and unable to access each other
+ files and directories. Similar requirements apply to other system non-file securable
+ objects like named pipes, critical sections, LPC queues, shared memory etc.
+
+
+ ** Linux Secure Container Executor
+
+ On Linux environment the secure container executor is the <<>>.
+ It uses an external program called the <>> to launch the container.
+ This program has the <<>> access right flag set which allows it to launch
+ the container with the permissions of the YARN application user.
+
+ *** Configuration
+
+ The configured directories for <<>> and
+ <<>> must be owned by the configured NodeManager user
+ (<<>>) and group (<<>>). The permission set on these directories must
+ be <<>>.
+
+ The <<>> program must be owned by <<>> and have the
+ permission set <<<---sr-s--->>>.
+
+ To configure the <<>> to use the <<>> set the following
+ in the <>:
+
++---+
+
+ yarn.nodemanager.container-executor.class
+ org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor
+
+
+
+ yarn.nodemanager.linux-container-executor.group
+ hadoop
+
++---+
+
+ Additionally the LCE requires the <<>> file, which is read by the
+ <<>> program.
+
++---+
+yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodemanager.linux-container-executor.group
+banned.users=#comma separated list of users who can not run applications
+allowed.system.users=#comma separated list of allowed system users
+min.user.id=1000#Prevent other super-users
++---+
+
+
+ ** Windows Secure Container Executor (WSCE)
+
+ The Windows environment secure container executor is the <<>>.
+ It uses the Windows S4U infrastructure to launch the container as the
+ YARN application user. The WSCE requires the presense of the <<>> service. This services
+ is hosted by <<<%HADOOP_HOME%\bin\winutils.exe>>> started with the <<>> command line argument. This
+ service offers some privileged operations that require LocalSystem authority so that the NM is not required
+ to run the entire JVM and all the NM code in an elevated context. The NM interacts with the <<>>
+ service by means of Local RPC (LRPC) via calls JNI to the RCP client hosted in <<>>.
+
+ *** Configuration
+
+ To configure the <<>> to use the <<>>
+ set the following in the <>:
+
++---+
+
+ yarn.nodemanager.container-executor.class
+ org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor
+
+
+
+ yarn.nodemanager.windows-secure-container-executor.group
+ yarn
+
++---+
+ *** wsce-site.xml
+
+ The hadoopwinutilsvc uses <<<%HADOOP_HOME%\etc\hadoop\wsce_site.xml to configure access to the privileged operations.
+
++---+
+
+ yarn.nodemanager.windows-secure-container-executor.impersonate.allowed
+ HadoopUsers
+
+
+
+ yarn.nodemanager.windows-secure-container-executor.impersonate.denied
+ HadoopServices,Administrators
+
+
+
+ yarn.nodemanager.windows-secure-container-executor.allowed
+ nodemanager
+
++---+
+
+ <<>> should contain the name of the service account running the
+ nodemanager. This user will be allowed to access the hadoopwintuilsvc functions.
+
+ <<>> should contain the users that are allowed to create
+ containers in the cluster. These users will be allowed to be impersonated by hadoopwinutilsvc.
+
+ <<>> should contain users that are explictly forbiden from
+ creating containers. hadoopwinutilsvc will refuse to impersonate these users.
+
+ *** Useful Links
+
+ * {{{http://msdn.microsoft.com/en-us/magazine/cc188757.aspx}Exploring S4U Kerberos Extensions in Windows Server 2003}}
+
+ * {{{https://issues.apache.org/jira/browse/YARN-1063}Winutils needs ability to create task as domain user}}
+
+ * {{{https://issues.apache.org/jira/browse/YARN-1972}Implement secure Windows Container Executor}}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
index 9f07b19..adb2a0a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
@@ -52,6 +52,8 @@ MapReduce NextGen aka YARN aka MRv2
* {{{./WebApplicationProxy.html}Web Application Proxy}}
* {{{./TimelineServer.html}YARN Timeline Server}}
+
+ * {{{./SecureContainer.html}YARN Secure Containers}}
* {{{../../hadoop-project-dist/hadoop-common/CLIMiniCluster.html}CLI MiniCluster}}