diff --git hadoop-common-project/hadoop-common/pom.xml hadoop-common-project/hadoop-common/pom.xml index fe8aba1..29a78b7 100644 --- hadoop-common-project/hadoop-common/pom.xml +++ hadoop-common-project/hadoop-common/pom.xml @@ -34,6 +34,8 @@ src/test/resources/kdc common true + ../etc/hadoop + wsce-site.xml @@ -681,6 +683,9 @@ /nologo /p:Configuration=Release /p:OutDir=${project.build.directory}/bin/ + /p:IntermediateOutputPath=${project.build.directory}/winutils/ + /p:WsceConfigDir=${wsce.config.dir} + /p:WsceConfigFile=${wsce.config.file} diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index fafa295..f8e9edf 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -17,11 +17,14 @@ */ package org.apache.hadoop.io.nativeio; +import java.io.BufferedInputStream; import java.io.File; import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.io.RandomAccessFile; import java.lang.reflect.Field; import java.nio.ByteBuffer; @@ -845,4 +848,71 @@ private static native void renameTo0(String src, String dst) private static native void link0(String src, String dst) throws NativeIOException; + + /** + * Wraps a process started by the winutils service helper. + * + */ + public static class WinutilsProcessStub extends Process { + + private final long hProcess; + private final long hThread; + private boolean disposed = false; + + private final InputStream stdErr; + private final InputStream stdOut; + private final OutputStream stdIn; + + public WinutilsProcessStub(long hProcess, long hThread, long hStdIn, long hStdOut, long hStdErr) { + this.hProcess = hProcess; + this.hThread = hThread; + + this.stdIn = new FileOutputStream(getFileDescriptorFromHandle(hStdIn)); + this.stdOut = new FileInputStream(getFileDescriptorFromHandle(hStdOut)); + this.stdErr = new FileInputStream(getFileDescriptorFromHandle(hStdErr)); + } + + private static native FileDescriptor getFileDescriptorFromHandle(long handle); + + @Override + public native void destroy(); + + @Override + public native int exitValue(); + + @Override + public InputStream getErrorStream() { + return stdErr; + } + @Override + public InputStream getInputStream() { + return stdOut; + } + @Override + public OutputStream getOutputStream() { + return stdIn; + } + @Override + public native int waitFor() throws InterruptedException; + + public synchronized native void dispose(); + + public native void resume() throws NativeIOException; + } + + public synchronized static WinutilsProcessStub createTaskAsUser( + String cwd, String jobName, String user, String pidFile, String cmdLine) + throws IOException { + if (!nativeLoaded) { + throw new IOException("NativeIO libraries are required for createTaskAsUser"); + } + synchronized(Shell.WindowsProcessLaunchLock) { + return createTaskAsUser0(cwd, jobName, user, pidFile, cmdLine); + } + } + + private static native WinutilsProcessStub createTaskAsUser0( + String cwd, String jobName, String user, String pidFile, String cmdLine) + throws NativeIOException; + } diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index fcdc021..67297cd 100644 --- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -643,6 +643,18 @@ public String toString() { } } + public interface ICommandExecutor { + + void execute() throws IOException; + + int getExitCode() throws IOException; + + String getOutput() throws IOException; + + void dispose(); + + } + /** * A simple shell command executor. * @@ -651,7 +663,7 @@ public String toString() { * directory and the environment remains unchanged. The output of the command * is stored as-is and is expected to be small. */ - public static class ShellCommandExecutor extends Shell { + public static class ShellCommandExecutor extends Shell implements ICommandExecutor { private String[] command; private StringBuffer output; @@ -743,6 +755,10 @@ public String toString() { } return builder.toString(); } + + @Override + public void dispose() { + } } /** diff --git hadoop-common-project/hadoop-common/src/main/native/native.vcxproj hadoop-common-project/hadoop-common/src/main/native/native.vcxproj index 0d67e1e..e743788 100644 --- hadoop-common-project/hadoop-common/src/main/native/native.vcxproj +++ hadoop-common-project/hadoop-common/src/main/native/native.vcxproj @@ -99,6 +99,7 @@ + diff --git hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index d8538c8..df57e0e 100644 --- hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -49,6 +49,7 @@ #include "file_descriptor.h" #include "errno_enum.h" +#include "winutils_process_stub.h" #define MMAP_PROT_READ org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_READ #define MMAP_PROT_WRITE org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_WRITE @@ -68,8 +69,13 @@ static jmethodID nioe_ctor; // Please see HADOOP-7156 for details. jobject pw_lock_object; +/* + * Throw a java.IO.IOException, generating the message from errno. + * NB. this is also used form winutils_process_stub.c + */ +extern void throw_ioe(JNIEnv* env, int errnum); + // Internal functions -static void throw_ioe(JNIEnv* env, int errnum); #ifdef UNIX static ssize_t get_pw_buflen(); #endif @@ -224,6 +230,12 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_initNative( errno_enum_init(env); PASS_EXCEPTIONS_GOTO(env, error); #endif + +#ifdef WINDOWS + winutils_process_stub_init(env); + PASS_EXCEPTIONS_GOTO(env, error); +#endif + return; error: // these are all idempodent and safe to call even if the @@ -236,6 +248,9 @@ error: #ifdef UNIX errno_enum_deinit(env); #endif +#ifdef WINDOWS + winutils_process_stub_deinit(env); +#endif } /* @@ -799,7 +814,7 @@ cleanup: /* * Throw a java.IO.IOException, generating the message from errno. */ -static void throw_ioe(JNIEnv* env, int errnum) +void throw_ioe(JNIEnv* env, int errnum) { #ifdef UNIX char message[80]; @@ -1142,6 +1157,86 @@ JNIEnv *env, jclass clazz) #endif } + +/* + * Class: org_apache_hadoop_io_nativeio_NativeIO + * Method: createTaskAsUser + * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String)Lorg/apache/hadoop/io/nativeio/NativeIO$WinutilsProcessStub + */ +JNIEXPORT jobject JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_createTaskAsUser0(JNIEnv* env, + jclass clazz, jstring jcwd, jstring jjobName, jstring juser, jstring jpidFile, jstring jcmdLine) { +#ifdef UNIX + THROW(env, "java/io/IOException", + "The function createTaskAsUser is not supported on Unix"); + return NULL; +#endif + +#ifdef WINDOWS + LPCWSTR cwd = NULL, jobName = NULL, + user = NULL, pidFile = NULL, cmdLine = NULL; + DWORD dwError = ERROR_SUCCESS; + HANDLE hProcess = INVALID_HANDLE_VALUE, + hThread = INVALID_HANDLE_VALUE, + hStdIn = INVALID_HANDLE_VALUE, + hStdOut = INVALID_HANDLE_VALUE, + hStdErr = INVALID_HANDLE_VALUE; + jobject ret = NULL; + + cwd = (LPCWSTR) (*env)->GetStringChars(env, jcwd, NULL); + if (!cwd) goto done; // exception was thrown + + jobName = (LPCWSTR) (*env)->GetStringChars(env, jjobName, NULL); + if (!jobName) goto done; // exception was thrown + + user = (LPCWSTR) (*env)->GetStringChars(env, juser, NULL); + if (!user) goto done; // exception was thrown + + pidFile = (LPCWSTR) (*env)->GetStringChars(env, jpidFile, NULL); + if (!pidFile) goto done; // exception was thrown + + cmdLine = (LPCWSTR) (*env)->GetStringChars(env, jcmdLine, NULL); + if (!cmdLine) goto done; // exception was thrown + + LogDebugMessage(L"createTaskAsUser: jcwd:%s job:%s juser:%s pid:%s cmd:%s\n", + cwd, jobName, user, pidFile, cmdLine); + + dwError = RpcCall_TaskCreateAsUser(cwd, jobName, user, pidFile, cmdLine, + &hProcess, &hThread, &hStdIn, &hStdOut, &hStdErr); + + if (ERROR_SUCCESS == dwError) { + ret = winutils_process_stub_create(env, (jlong) hProcess, (jlong) hThread, + (jlong) hStdIn, (jlong) hStdOut, (jlong) hStdErr); + + if (NULL == ret) { + TerminateProcess(hProcess, EXIT_FAILURE); + CloseHandle(hThread); + CloseHandle(hProcess); + CloseHandle(hStdIn); + CloseHandle(hStdOut); + CloseHandle(hStdErr); + } + } + + + if (dwError != ERROR_SUCCESS) { + throw_ioe (env, dwError); + } + +done: + + if (cwd) (*env)->ReleaseStringChars(env, jcwd, cwd); + if (jobName) (*env)->ReleaseStringChars(env, jjobName, jobName); + if (user) (*env)->ReleaseStringChars(env, juser, user); + if (pidFile) (*env)->ReleaseStringChars(env, jpidFile, pidFile); + if (cmdLine) (*env)->ReleaseStringChars(env, jcmdLine, cmdLine); + + return ret; + +#endif +} + + /** * vim: sw=2: ts=2: et: */ diff --git hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.c hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.c new file mode 100644 index 0000000..049af45 --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.c @@ -0,0 +1,198 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include +#include "org_apache_hadoop.h" +#include "winutils_process_stub.h" +#include "winutils.h" +#include "file_descriptor.h" + +// class of org.apache.hadoop.io.nativeio.NativeIO.WinutilsProcessStub +static jclass wps_class = NULL; + + +static jmethodID wps_constructor = NULL; +static jfieldID wps_hProcess = NULL; +static jfieldID wps_hThread = NULL; +static jfieldID wps_disposed = NULL; + +extern void throw_ioe(JNIEnv* env, int errnum); + +void winutils_process_stub_init(JNIEnv *env) { + if (wps_class != NULL) return; // already initted + + wps_class = (*env)->FindClass(env, WINUTILS_PROCESS_STUB_CLASS); + PASS_EXCEPTIONS(env); + + wps_class = (*env)->NewGlobalRef(env, wps_class); + PASS_EXCEPTIONS(env); + + wps_hProcess = (*env)->GetFieldID(env, wps_class, "hProcess", "J"); + PASS_EXCEPTIONS(env); + + wps_hThread = (*env)->GetFieldID(env, wps_class, "hThread", "J"); + PASS_EXCEPTIONS(env); + + wps_disposed = (*env)->GetFieldID(env, wps_class, "disposed", "Z"); + PASS_EXCEPTIONS(env); + + wps_constructor = (*env)->GetMethodID(env, wps_class, "", "(JJJJJ)V"); + PASS_EXCEPTIONS(env); + + LogDebugMessage(L"winutils_process_stub_init\n"); +} + +void winutils_process_stub_deinit(JNIEnv *env) { + if (wps_class != NULL) { + (*env)->DeleteGlobalRef(env, wps_class); + wps_class = NULL; + } + wps_hProcess = NULL; + wps_hThread = NULL; + wps_disposed = NULL; + wps_constructor = NULL; + LogDebugMessage(L"winutils_process_stub_deinit\n"); +} + +jobject winutils_process_stub_create(JNIEnv *env, + jlong hProcess, jlong hThread, jlong hStdIn, jlong hStdOut, jlong hStdErr) { + jobject obj = (*env)->NewObject(env, wps_class, wps_constructor, + hProcess, hThread, hStdIn, hStdOut, hStdErr); + PASS_EXCEPTIONS_RET(env, NULL); + + LogDebugMessage(L"winutils_process_stub_create: %p\n", obj); + + return obj; +} + + +/* + * native void destroy(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_destroy( + JNIEnv *env, jobject objSelf) { + + HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); + LogDebugMessage(L"TerminateProcess: %x\n", hProcess); + TerminateProcess(hProcess, EXIT_FAILURE); +} + +/* + * native void waitFor(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_waitFor( + JNIEnv *env, jobject objSelf) { + + HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); + LogDebugMessage(L"WaitForSingleObject: %x\n", hProcess); + WaitForSingleObject(hProcess, INFINITE); +} + + + +/* + * native void resume(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_resume( + JNIEnv *env, jobject objSelf) { + + DWORD dwError; + HANDLE hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread); + if (-1 == ResumeThread(hThread)) { + dwError = GetLastError(); + LogDebugMessage(L"ResumeThread: %x error:%d\n", hThread, dwError); + throw_ioe(env, dwError); + } +} + +/* + * native int exitValue(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT jint JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_exitValue( + JNIEnv *env, jobject objSelf) { + + DWORD exitCode; + DWORD dwError; + HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); + if (!GetExitCodeProcess(hProcess, &exitCode)) { + dwError = GetLastError(); + throw_ioe(env, dwError); + return dwError; // exception was thrown, return value doesn't really matter + } + LogDebugMessage(L"GetExitCodeProcess: %x :%d\n", hProcess, exitCode); + + return exitCode; +} + + +/* + * native void dispose(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_dispose( + JNIEnv *env, jobject objSelf) { + + HANDLE hProcess = INVALID_HANDLE_VALUE, + hThread = INVALID_HANDLE_VALUE; + + jboolean disposed = (*env)->GetBooleanField(env, objSelf, wps_disposed); + + if (JNI_TRUE != disposed) { + hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); + hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread); + + CloseHandle(hProcess); + CloseHandle(hThread); + (*env)->SetBooleanField(env, objSelf, wps_disposed, JNI_TRUE); + LogDebugMessage(L"disposed: %p\n", objSelf); + } +} + + +/* + * native static FileDescriptor getFileDescriptorFromHandle(long handle); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT jobject JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_getFileDescriptorFromHandle( + JNIEnv *env, jclass klass, jlong handle) { + + LogDebugMessage(L"getFileDescriptorFromHandle: %x\n", handle); + return fd_create(env, (long) handle); +} + diff --git hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.h hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.h new file mode 100644 index 0000000..6ab8ad6 --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.h @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + + +#define WINUTILS_PROCESS_STUB_CLASS "org/apache/hadoop/io/nativeio/NativeIO$WinutilsProcessStub" + +void winutils_process_stub_init(JNIEnv *env); +void winutils_process_stub_deinit(JNIEnv *env); +jobject winutils_process_stub_create(JNIEnv *env, + jlong hProcess, jlong hThread, jlong hStdIn, jlong hStdOut, jlong hStdErr); + + diff --git hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h index 92a6b27..3fd5a58 100644 --- hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h +++ hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h @@ -32,6 +32,7 @@ #define UNIX #endif + /* A helper macro to 'throw' a java exception. */ #define THROW(env, exception_name, message) \ { \ diff --git hadoop-common-project/hadoop-common/src/main/winutils/client.c hadoop-common-project/hadoop-common/src/main/winutils/client.c new file mode 100644 index 0000000..bc8efcd --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/winutils/client.c @@ -0,0 +1,161 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" +#include +#include +#include "hadoopwinutilsvc_h.h" + +#pragma comment(lib, "Rpcrt4.lib") +#pragma comment(lib, "advapi32.lib") + +static ACCESS_MASK CLIENT_MASK = 1; + +VOID ReportClientError(LPWSTR lpszLocation, DWORD dwError) { + LPWSTR debugMsg = NULL; + int len; + WCHAR hexError[32]; + HRESULT hr; + + if (IsDebuggerPresent()) { + len = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, dwError, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR)&debugMsg, 0, NULL); + + LogDebugMessage(L"%s: %s: %x: %.*s\n", GetSystemTimeString(), lpszLocation, dwError, len, debugMsg); + } + + if (NULL != debugMsg) LocalFree(debugMsg); +} + +DWORD RpcCall_TaskCreateAsUser( + LPCWSTR cwd, LPCWSTR jobName, + LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine, + HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr) +{ + DWORD dwError = EXIT_FAILURE; + RPC_STATUS status; + LPWSTR lpszStringBinding = NULL; + ULONG ulCode; + DWORD dwSelfPid = GetCurrentProcessId(); + CREATE_PROCESS_REQUEST request; + CREATE_PROCESS_RESPONSE *response = NULL; + RPC_SECURITY_QOS_V3 qos; + PSID pLocalSystemSid = NULL; + SID_IDENTIFIER_AUTHORITY authNT = SECURITY_NT_AUTHORITY; + + if (!AllocateAndInitializeSid(&authNT, 1, + SECURITY_LOCAL_SYSTEM_RID, + 0, 0, 0, 0, 0, 0, 0, + &pLocalSystemSid)) { + dwError = GetLastError(); + ReportClientError(L"AllocateAndInitializeSid", dwError); + goto done; + } + + ZeroMemory(&qos, sizeof(qos)); + qos.Version = RPC_C_SECURITY_QOS_VERSION_3; + qos.Capabilities = RPC_C_QOS_CAPABILITIES_LOCAL_MA_HINT | RPC_C_QOS_CAPABILITIES_MUTUAL_AUTH; + qos.IdentityTracking = RPC_C_QOS_IDENTITY_DYNAMIC; + qos.ImpersonationType = RPC_C_IMP_LEVEL_DEFAULT; + qos.Sid = pLocalSystemSid; + + ZeroMemory(&request, sizeof(request)); + request.cwd = cwd; + request.jobName = jobName; + request.user = user; + request.pidFile = pidFile; + request.cmdLine = cmdLine; + + status = RpcStringBindingCompose(NULL, + SVCBINDING, + NULL, + SVCNAME, + NULL, + &lpszStringBinding); + if (RPC_S_OK != status) { + ReportClientError(L"RpcStringBindingCompose", status); + dwError = status; + goto done; + } + + status = RpcBindingFromStringBinding(lpszStringBinding, &hHadoopWinutilsSvcBinding); + + if (RPC_S_OK != status) { + ReportClientError(L"RpcBindingFromStringBinding", status); + dwError = status; + goto done; + } + + status = RpcBindingSetAuthInfoEx( + hHadoopWinutilsSvcBinding, + NULL, + RPC_C_AUTHN_LEVEL_PKT_PRIVACY, // AuthnLevel + RPC_C_AUTHN_WINNT, // AuthnSvc + NULL, // AuthnIdentity (self) + RPC_C_AUTHZ_NONE, // AuthzSvc + &qos); + if (RPC_S_OK != status) { + ReportClientError(L"RpcBindingSetAuthInfoEx", status); + dwError = status; + goto done; + } + + RpcTryExcept { + dwError = WinutilsCreateProcessAsUser(dwSelfPid, &request, &response); + } + RpcExcept(1) { + ulCode = RpcExceptionCode(); + ReportClientError(L"RpcExcept", ulCode); + dwError = (DWORD) ulCode; + } + RpcEndExcept; + + if (ERROR_SUCCESS == dwError) { + *phProcess = response->hProcess; + *phThread = response->hThread; + *phStdIn = response->hStdIn; + *phStdOut = response->hStdOut; + *phStdErr = response->hStdErr; + } + + // From here on forward we do no change dwError even on RPC cleanup errors + status = RpcBindingFree(&hHadoopWinutilsSvcBinding); + if (RPC_S_OK != status) { + ReportClientError(L"RpcBindingFree", status); + goto done; + } + +done: + if (pLocalSystemSid) FreeSid(pLocalSystemSid); + + if (NULL != response) { + MIDL_user_free(response); + } + + if (NULL != lpszStringBinding) { + status = RpcStringFree(&lpszStringBinding); + if (RPC_S_OK != status) { + ReportClientError(L"RpcStringFree", status); + } + } + + return dwError; +} + diff --git hadoop-common-project/hadoop-common/src/main/winutils/config.cpp hadoop-common-project/hadoop-common/src/main/winutils/config.cpp new file mode 100644 index 0000000..1e07b7f --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/winutils/config.cpp @@ -0,0 +1,174 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" +#include +#include +#import "msxml6.dll" + +#define ERROR_CHECK_HRESULT_DONE(hr, message) \ + if (FAILED(hr)) { \ + dwError = (DWORD) hr; \ + LogDebugMessage(L"%s: %x", message, hr); \ + goto done; \ + } + +DWORD BuildPathRelativeToModule( + __in LPCWSTR relativePath, + __in size_t len, + __out_ecount(len) LPWSTR buffer) { + DWORD dwError = ERROR_SUCCESS; + WCHAR moduleFile[MAX_PATH]; + WCHAR modulePath[_MAX_DIR]; + WCHAR moduleDrive[_MAX_DRIVE]; + DWORD size; + HRESULT hr = S_OK; + errno_t errno; + + size = GetModuleFileName(NULL, moduleFile, MAX_PATH); + dwError = GetLastError(); // Always check due to ERROR_INSUFFICIENT_BUFFER + if (dwError) { + LogDebugMessage(L"GetModuleFileName: %x\n", dwError); + goto done; + } + + errno = _wsplitpath_s(moduleFile, + moduleDrive, _MAX_DRIVE, + modulePath, _MAX_DIR, + NULL, 0, // fname, not interesting + NULL, 0); // extenssion, not interesting + if (errno) { + LogDebugMessage(L"_wsplitpath_s: %x\n", errno); + dwError = ERROR_BAD_PATHNAME; + goto done; + } + + hr = StringCbPrintf(buffer, len, L"%s%s%s", moduleDrive, modulePath, relativePath); + if (FAILED(hr)) { + // There is no reliable HRESULT to WIN32 mapping, use code. + // see http://blogs.msdn.com/b/oldnewthing/archive/2006/11/03/942851.aspx + // + dwError = HRESULT_CODE(hr); + goto done; + } + + LogDebugMessage(L"BuildPathRelativeToModule: %s (%s)\n", buffer, relativePath); + +done: + return dwError; +} + +DWORD GetConfigValue( + __in LPCWSTR relativePath, + __in LPCWSTR keyName, + __out size_t* len, __out_ecount(len) LPCWSTR* value) { + + DWORD dwError = ERROR_SUCCESS; + WCHAR xmlPath[MAX_PATH]; + + *len = 0; + *value = NULL; + + dwError = BuildPathRelativeToModule( + relativePath, + sizeof(xmlPath)/sizeof(WCHAR), + xmlPath); + + if (dwError) { + goto done; + } + + dwError = GetConfigValueFromXmlFile(xmlPath, keyName, len, value); + +done: + if (*len) { + LogDebugMessage(L"GetConfigValue:%d key:%s len:%d value:%.*s from:%s\n", dwError, keyName, *len, *len, *value, xmlPath); + } + return dwError; +} + + +DWORD GetConfigValueFromXmlFile(__in LPCWSTR xmlFile, __in LPCWSTR keyName, + __out size_t* outLen, __out_ecount(len) LPCWSTR* outValue) { + + DWORD dwError = ERROR_SUCCESS; + HRESULT hr; + WCHAR keyXsl[8192]; + size_t len = 0; + LPWSTR value = NULL; + BOOL comInitialized = FALSE; + + *outLen = 0; + *outValue = NULL; + + hr = CoInitialize(NULL); + ERROR_CHECK_HRESULT_DONE(hr, L"CoInitialize"); + comInitialized = TRUE; + + hr = StringCbPrintf(keyXsl, sizeof(keyXsl), L"//configuration/property[name='%s']/value/text()", keyName); + ERROR_CHECK_HRESULT_DONE(hr, L"StringCbPrintf"); + + try { + MSXML2::IXMLDOMDocument2Ptr pDoc; + hr = pDoc.CreateInstance(__uuidof(MSXML2::DOMDocument60), NULL, CLSCTX_INPROC_SERVER); + ERROR_CHECK_HRESULT_DONE(hr, L"CreateInstance"); + + pDoc->async = VARIANT_FALSE; + pDoc->validateOnParse = VARIANT_FALSE; + pDoc->resolveExternals = VARIANT_FALSE; + + _variant_t file(xmlFile); + + if (VARIANT_FALSE == pDoc->load(file)) { + dwError = pDoc->parseError->errorCode; + LogDebugMessage(L"load %s failed:%d %s\n", xmlFile, dwError, + static_cast(pDoc->parseError->Getreason())); + goto done; + } + + MSXML2::IXMLDOMElementPtr pRoot = pDoc->documentElement; + MSXML2::IXMLDOMNodePtr keyNode = pRoot->selectSingleNode(keyXsl); + + if (keyNode) { + _bstr_t bstrValue = static_cast<_bstr_t>(keyNode->nodeValue); + len = bstrValue.length(); + value = (LPWSTR) LocalAlloc(LPTR, (len+1) * sizeof(WCHAR)); + LPCWSTR lpwszValue = static_cast(bstrValue); + memcpy(value, lpwszValue, (len) * sizeof(WCHAR)); + LogDebugMessage(L"key:%s :%.*s [%s]\n", keyName, len, value, lpwszValue); + *outLen = len; + *outValue = value; + } + else { + LogDebugMessage(L"node Xpath:%s not found in:%s\n", keyXsl, xmlFile); + } + } + catch(_com_error errorObject) { + dwError = errorObject.Error(); + LogDebugMessage(L"catch _com_error:%x %s\n", dwError, errorObject.ErrorMessage()); + goto done; + } + +done: + if (comInitialized) { + CoUninitialize(); + } + + return dwError; +} + + diff --git hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl new file mode 100644 index 0000000..2285178 --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl @@ -0,0 +1,35 @@ +import "oaidl.idl"; +import "ocidl.idl"; + +[ + uuid(0492311C-1718-4F53-A6EB-86AD7039988D), + version(1.0), + pointer_default(unique), + implicit_handle(handle_t hHadoopWinutilsSvcBinding), + endpoint("ncalrpc:[hadoopwinutilsvc]"), +] +interface HadoopWinutilSvc +{ + typedef struct { + [string] const wchar_t* cwd; + [string] const wchar_t* jobName; + [string] const wchar_t* user; + [string] const wchar_t* pidFile; + [string] const wchar_t* cmdLine; + } CREATE_PROCESS_REQUEST; + + typedef struct { + LONG_PTR hProcess; + LONG_PTR hThread; + LONG_PTR hStdIn; + LONG_PTR hStdOut; + LONG_PTR hStdErr; + } CREATE_PROCESS_RESPONSE; + + + error_status_t WinutilsCreateProcessAsUser( + [in] int nmPid, + [in] CREATE_PROCESS_REQUEST *request, + [out] CREATE_PROCESS_RESPONSE **response); + +} \ No newline at end of file diff --git hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h index bae754c..51835b6 100644 --- hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h +++ hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h @@ -30,6 +30,10 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + enum EXIT_CODE { /* Common success exit code shared among all utilities */ @@ -178,3 +182,51 @@ DWORD LoadUserProfileForLogon(__in HANDLE logonHandle, __out PROFILEINFO * pi); DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi); +DWORD RunService(__in int argc, __in_ecount(argc) wchar_t *argv[]); +void ServiceUsage(); + +LPCWSTR GetSystemTimeString(); + +VOID LogDebugMessage(LPCWSTR format, ...); + +DWORD SplitStringIgnoreSpaceW( + __in size_t len, + __in_ecount(len) LPCWSTR source, + __in WCHAR deli, + __out size_t* count, __out_ecount(count) WCHAR*** out); + +DWORD GetConfigValue( + __in LPCWSTR relativePath, + __in LPCWSTR keyName, + __out size_t* len, + __out_ecount(len) LPCWSTR* value); +DWORD GetConfigValueFromXmlFile( + __in LPCWSTR xmlFile, + __in LPCWSTR keyName, + __out size_t* len, + __out_ecount(len) LPCWSTR* value); + + +DWORD BuildServiceSecurityDescriptor( + __in ACCESS_MASK accessMask, + __in size_t grantSidCount, + __in_ecount(grantSidCount) PSID* pGrantSids, + __in size_t denySidCount, + __in_ecount(denySidCount) PSID* pDenySids, + __out PSECURITY_DESCRIPTOR* pSD); + +extern const WCHAR* wsceConfigRelativePath; + +#define SVCNAME TEXT("hadoopwinutilsvc") +#define SVCBINDING TEXT("ncalrpc") + +int RpcCall_TaskCreateAsUser( + LPCWSTR cwd, LPCWSTR jobName, + LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine, + HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr); + +#ifdef __cplusplus +} +#endif + + diff --git hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c index da16ff5..19aae4c 100644 --- hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c +++ hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c @@ -19,9 +19,24 @@ #pragma comment(lib, "netapi32.lib") #pragma comment(lib, "Secur32.lib") #pragma comment(lib, "Userenv.lib") +#pragma comment(lib, "Ntdsapi.lib") + #include "winutils.h" +#include +#include #include #include +#include +#include + +#define WIDEN_STRING(x) WIDEN_STRING_(x) +#define WIDEN_STRING_(x) L ## x +#define STRINGIFY(x) STRINGIFY_(x) +#define STRINGIFY_(x) #x + + +#pragma message("WSCE config is " STRINGIFY(WSCE_CONFIG_DIR) "\\" STRINGIFY(WSCE_CONFIG_FILE)) +const WCHAR* wsceConfigRelativePath = WIDEN_STRING(STRINGIFY(WSCE_CONFIG_DIR)) L"\\" WIDEN_STRING(STRINGIFY(WSCE_CONFIG_FILE)); /* * The array of 12 months' three-letter abbreviations @@ -1706,12 +1721,15 @@ void ReportErrorCode(LPCWSTR func, DWORD err) (LPWSTR)&msg, 0, NULL); if (len > 0) { + LogDebugMessage(L"%s error (%d): %s\n", func, err, msg); fwprintf(stderr, L"%s error (%d): %s\n", func, err, msg); } else { + LogDebugMessage(L"%s error code: %d.\n", func, err); fwprintf(stderr, L"%s error code: %d.\n", func, err); } + if (msg != NULL) LocalFree(msg); } @@ -2026,6 +2044,8 @@ done: return loadProfileStatus; } + + DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi) { DWORD touchProfileStatus = ERROR_ASSERTION_FAILURE; // Failure to set status should trigger error @@ -2046,3 +2066,384 @@ DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi) done: return touchProfileStatus; } + + +LPCWSTR GetSystemTimeString() { + __declspec(thread) static WCHAR buffer[1024]; + DWORD dwError; + FILETIME ftime; + SYSTEMTIME systime; + LARGE_INTEGER counter, frequency; + int subSec; + double qpc; + HRESULT hr; + buffer[0] = L'\0'; + + // GetSystemTimePreciseAsFileTime is only available in Win8+ and our libs do not link against it + + GetSystemTimeAsFileTime(&ftime); + + if (!FileTimeToSystemTime(&ftime, &systime)) { + dwError = GetLastError(); + LogDebugMessage(L"FileTimeToSystemTime error:%d\n", dwError); + goto done; + } + + // Get the ms from QPC. GetSystemTimeAdjustment is ignored... + + QueryPerformanceCounter(&counter); + QueryPerformanceFrequency(&frequency); + + qpc = (double) counter.QuadPart / (double) frequency.QuadPart; + subSec = ((qpc - (long)qpc) * 1000000); + + hr = StringCbPrintf(buffer, sizeof(buffer), L"%02d:%02d:%02d.%06d", + (int)systime.wHour, (int)systime.wMinute, (int)systime.wSecond, (int)subSec); + + if (FAILED(hr)) { + LogDebugMessage(L"StringCbPrintf error:%d\n", hr); + } +done: + return buffer; +} + + +//---------------------------------------------------------------------------- +// Function: LogDebugMessage +// +// Description: +// Sends a message to the debugger console, if one is attached +// +// Notes: +// Native debugger: windbg, ntsd, cdb, visual studio +// +VOID LogDebugMessage(LPCWSTR format, ...) { + LPWSTR buffer[8192]; + va_list args; + HRESULT hr; + + if (!IsDebuggerPresent()) return; + + va_start(args, format); + hr = StringCbVPrintf(buffer, sizeof(buffer), format, args); + if (SUCCEEDED(hr)) { + OutputDebugString(buffer); + } + va_end(args); +} + +//---------------------------------------------------------------------------- +// Function: SplitStringIgnoreSpaceW +// +// Description: +// splits a null-terminated string based on a delimiter +// +// Returns: +// ERROR_SUCCESS: on success +// error code: otherwise +// +// Notes: +// The tokes are also null-terminated +// Caller should use LocalFree to clear outTokens +// +DWORD SplitStringIgnoreSpaceW( + __in size_t len, + __in_ecount(len) LPCWSTR source, + __in WCHAR deli, + __out size_t* count, + __out_ecount(count) WCHAR*** outTokens) { + + size_t tokenCount = 0; + size_t crtSource; + size_t crtToken = 0; + WCHAR* lpwszTokenStart = NULL; + WCHAR* lpwszTokenEnd = NULL; + WCHAR* lpwszBuffer = NULL; + size_t tokenLength = 0; + size_t cchBufferLength = 0; + WCHAR crt; + WCHAR** tokens = NULL; + enum {BLANK, TOKEN, DELIMITER} State = BLANK; + + for(crtSource = 0; crtSource < len; ++crtSource) { + crt = source[crtSource]; + switch(State) { + case BLANK: // intentional fallthrough + case DELIMITER: + if (crt == deli) { + State = DELIMITER; + } + else if (!iswspace(crt)) { + ++tokenCount; + lpwszTokenEnd = lpwszTokenStart = source + crtSource; + State = TOKEN; + } + else { + State = BLANK; + } + break; + case TOKEN: + if (crt == deli) { + State = DELIMITER; + cchBufferLength += lpwszTokenEnd - lpwszTokenStart + 2; + } + else if (!iswspace(crt)) { + lpwszTokenEnd = source + crtSource; + } + break; + } + } + + if (State == TOKEN) { + cchBufferLength += lpwszTokenEnd - lpwszTokenStart + 2; + } + + LogDebugMessage(L"counted %d [buffer:%d] tokens in %s\n", tokenCount, cchBufferLength, source); + + #define COPY_CURRENT_TOKEN \ + tokenLength = lpwszTokenEnd - lpwszTokenStart + 1; \ + tokens[crtToken] = lpwszBuffer; \ + memcpy(tokens[crtToken], lpwszTokenStart, tokenLength*sizeof(WCHAR)); \ + tokens[crtToken][tokenLength] = L'\0'; \ + lpwszBuffer += (tokenLength+1); \ + ++crtToken; + + if (tokenCount) { + + // We use one contigous memory for both the pointer arrays and the data copy buffers + // We cannot use in-place references (zero-copy) because the function users + // need null-terminated strings for the tokens + + tokens = (WCHAR**) LocalAlloc(LPTR, + sizeof(WCHAR*) * tokenCount + // for the pointers + sizeof(WCHAR) * cchBufferLength); // for the data + + // Data will be copied after the array + lpwszBuffer = (WCHAR*)(((BYTE*)tokens) + (sizeof(WCHAR*) * tokenCount)); + + State = BLANK; + + for(crtSource = 0; crtSource < len; ++crtSource) { + crt = source[crtSource]; + switch(State) { + case DELIMITER: // intentional fallthrough + case BLANK: + if (crt == deli) { + State = DELIMITER; + } + else if (!iswspace(crt)) { + lpwszTokenEnd = lpwszTokenStart = source + crtSource; + State = TOKEN; + } + else { + State = BLANK; + } + break; + case TOKEN: + if (crt == deli) { + COPY_CURRENT_TOKEN; + State = DELIMITER; + } + else if (!iswspace(crt)) { + lpwszTokenEnd = source + crtSource; + } + break; + } + } + + // Copy out last token, if any + if (TOKEN == State) { + COPY_CURRENT_TOKEN; + } + } + + *count = tokenCount; + *outTokens = tokens; + + return ERROR_SUCCESS; +} + +//---------------------------------------------------------------------------- +// Function: BuildServiceSecurityDescriptor +// +// Description: +// Builds a security descriptor for an arbitrary object +// +// Returns: +// ERROR_SUCCESS: on success +// error code: otherwise +// +// Notes: +// The SD is a of the self-contained flavor (offsets, not pointers) +// Caller should use LocalFree to clear allocated pSD +// +DWORD BuildServiceSecurityDescriptor( + __in ACCESS_MASK accessMask, + __in size_t grantSidCount, + __in_ecount(grantSidCount) PSID* pGrantSids, + __in size_t denySidCount, + __in_ecount(denySidCount) PSID* pDenySids, + __out PSECURITY_DESCRIPTOR* pSD) { + + DWORD dwError = ERROR_SUCCESS; + int crt = 0; + int len = 0; + EXPLICIT_ACCESS* eas = NULL; + LPWSTR lpszSD = NULL; + ULONG cchSD = 0; + HANDLE hToken = INVALID_HANDLE_VALUE; + DWORD dwBufferSize = 0; + PTOKEN_USER pTokenUser = NULL; + PTOKEN_PRIMARY_GROUP pTokenGroup = NULL; + PSECURITY_DESCRIPTOR pTempSD = NULL; + ULONG cbSD = 0; + TRUSTEE owner, group; + + ZeroMemory(&owner, sizeof(owner)); + + // We'll need our own SID to add as SD owner + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &hToken)) { + dwError = GetLastError(); + goto done; + } + + if (!GetTokenInformation(hToken, TokenUser, NULL, 0, &dwBufferSize)) { + dwError = GetLastError(); + if (ERROR_INSUFFICIENT_BUFFER != dwError) { + goto done; + } + } + + pTokenUser = (PTOKEN_USER) LocalAlloc(LPTR, dwBufferSize); + if (NULL == pTokenUser) { + dwError = GetLastError(); + goto done; + } + + if (!GetTokenInformation(hToken, TokenUser, pTokenUser, dwBufferSize, &dwBufferSize)) { + dwError = GetLastError(); + goto done; + } + + if (!IsValidSid(pTokenUser->User.Sid)) { + dwError = ERROR_INVALID_PARAMETER; + goto done; + } + + dwBufferSize = 0; + if (!GetTokenInformation(hToken, TokenPrimaryGroup, NULL, 0, &dwBufferSize)) { + dwError = GetLastError(); + if (ERROR_INSUFFICIENT_BUFFER != dwError) { + goto done; + } + } + + pTokenGroup = (PTOKEN_USER) LocalAlloc(LPTR, dwBufferSize); + if (NULL == pTokenUser) { + dwError = GetLastError(); + goto done; + } + + if (!GetTokenInformation(hToken, TokenPrimaryGroup, pTokenGroup, dwBufferSize, &dwBufferSize)) { + dwError = GetLastError(); + goto done; + } + + if (!IsValidSid(pTokenGroup->PrimaryGroup)) { + dwError = ERROR_INVALID_PARAMETER; + goto done; + } + + owner.TrusteeForm = TRUSTEE_IS_SID; + owner.TrusteeType = TRUSTEE_IS_UNKNOWN; + owner.ptstrName = (LPCWSTR) pTokenUser->User.Sid; + + group.TrusteeForm = TRUSTEE_IS_SID; + group.TrusteeType = TRUSTEE_IS_UNKNOWN; + group.ptstrName = (LPCWSTR) pTokenGroup->PrimaryGroup; + + eas = (EXPLICIT_ACCESS*) alloca(sizeof(EXPLICIT_ACCESS) * (grantSidCount + denySidCount)); + + // Build the granted list + for (crt = 0; crt < grantSidCount; ++crt) { + eas[crt].grfAccessPermissions = accessMask; + eas[crt].grfAccessMode = GRANT_ACCESS; + eas[crt].grfInheritance = NO_INHERITANCE; + eas[crt].Trustee.TrusteeForm = TRUSTEE_IS_SID; + eas[crt].Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN; + eas[crt].Trustee.ptstrName = (LPCWSTR) pGrantSids[crt]; + } + + // Build the deny list + for (; crt < grantSidCount + denySidCount; ++crt) { + eas[crt].grfAccessPermissions = accessMask; + eas[crt].grfAccessMode = DENY_ACCESS; + eas[crt].grfInheritance = NO_INHERITANCE; + eas[crt].Trustee.TrusteeForm = TRUSTEE_IS_SID; + eas[crt].Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN; + eas[crt].Trustee.ptstrName = (LPCWSTR) pDenySids[crt - grantSidCount]; + } + + dwError = BuildSecurityDescriptor( + &owner, + &group, + crt, + eas, + 0, // cCountOfAuditEntries + NULL, // pListOfAuditEntries + NULL, // pOldSD + &cbSD, + &pTempSD); + if (ERROR_SUCCESS != dwError) { + goto done; + } + + *pSD = pTempSD; + pTempSD = NULL; + + if (IsDebuggerPresent()) { + ConvertSecurityDescriptorToStringSecurityDescriptor(*pSD, + SDDL_REVISION_1, + DACL_SECURITY_INFORMATION, + &lpszSD, + &cchSD); + LogDebugMessage(L"pSD: %.*s\n", cchSD, lpszSD); + } + +done: + if (pTokenUser) LocalFree(pTokenUser); + if (INVALID_HANDLE_VALUE != hToken) CloseHandle(hToken); + if (lpszSD) LocalFree(lpszSD); + if (pTempSD) LocalFree(pTempSD); + return dwError; +} + + +//---------------------------------------------------------------------------- +// Function: MIDL_user_allocate +// +// Description: +// Hard-coded function name used by RPC midl code for allocations +// +// Notes: +// Must match the de-allocation mechanism used in MIDL_user_free +// +void __RPC_FAR * __RPC_USER MIDL_user_allocate(size_t len) +{ + return LocalAlloc(LPTR, len); +} + + //---------------------------------------------------------------------------- + // Function: MIDL_user_free + // + // Description: + // Hard-coded function name used by RPC midl code for deallocations + // + // NoteS: + // Must match the allocation mechanism used in MIDL_user_allocate + // +void __RPC_USER MIDL_user_free(void __RPC_FAR * ptr) +{ + LocalFree(ptr); +} + diff --git hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj index fc0519d..cbc4ae9 100644 --- hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj +++ hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj @@ -1,5 +1,4 @@  - - - - Debug - Win32 - Debug x64 - - Release - Win32 - Release x64 @@ -42,22 +32,11 @@ winutils - - StaticLibrary - true - Unicode - StaticLibrary true Unicode - - StaticLibrary - false - true - Unicode - StaticLibrary false @@ -67,15 +46,9 @@ - - - - - - @@ -83,49 +56,26 @@ include;$(IncludePath) - - true - true - - ..\..\..\target\winutils\$(Configuration)\ - - - false false - ..\..\..\target\bin\ - ..\..\..\target\winutils\$(Platform)\$(Configuration)\ - - - - - Level3 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - - - Console - true - - Level4 Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + WIN32;_DEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions) Console true - + Level3 @@ -133,7 +83,7 @@ MaxSpeed true true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + WIN32;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions) Console @@ -142,29 +92,34 @@ true - + - Level3 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(IntermediateOutputPath) - - Console - true - true - true - + + true + X64 + $(IntermediateOutputPath) + true + true + true + 2 + + + + + + + + + diff --git hadoop-common-project/hadoop-common/src/main/winutils/main.c hadoop-common-project/hadoop-common/src/main/winutils/main.c index 0f40774..ac73aec 100644 --- hadoop-common-project/hadoop-common/src/main/winutils/main.c +++ hadoop-common-project/hadoop-common/src/main/winutils/main.c @@ -17,12 +17,27 @@ #include "winutils.h" +#include + static void Usage(LPCWSTR program); +LONG WINAPI WinutilsSehUnhandled(_In_ struct _EXCEPTION_POINTERS *ecxr) { + LogDebugMessage(L"unhandled SEH: code:%x flags:%d\n", + ecxr->ExceptionRecord->ExceptionCode, + ecxr->ExceptionRecord->ExceptionFlags); + fwprintf(stderr, L"Unhandled exception code:%x at address:%p", + ecxr->ExceptionRecord->ExceptionCode, + ecxr->ExceptionRecord->ExceptionAddress); + ExitProcess(ERROR_UNHANDLED_EXCEPTION); + return EXCEPTION_EXECUTE_HANDLER; // not that it matters... +} + int wmain(__in int argc, __in_ecount(argc) wchar_t* argv[]) { LPCWSTR cmd = NULL; + SetUnhandledExceptionFilter(WinutilsSehUnhandled); + if (argc < 2) { Usage(argv[0]); @@ -67,6 +82,10 @@ int wmain(__in int argc, __in_ecount(argc) wchar_t* argv[]) { return SystemInfo(); } + else if (wcscmp(L"service", cmd) == 0) + { + return RunService(argc - 1, argv + 1); + } else if (wcscmp(L"help", cmd) == 0) { Usage(argv[0]); @@ -119,5 +138,9 @@ The available commands and their usages are:\n\n", program); fwprintf(stdout, L"%-15s%s\n\n", L"task", L"Task operations."); TaskUsage(); + + fwprintf(stdout, L"%-15s%s\n\n", L"service", L"Service operations."); + ServiceUsage(); + fwprintf(stdout, L"\n\n"); } diff --git hadoop-common-project/hadoop-common/src/main/winutils/service.c hadoop-common-project/hadoop-common/src/main/winutils/service.c new file mode 100644 index 0000000..70c2a1d --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/winutils/service.c @@ -0,0 +1,884 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" +#include "winutils_msg.h" +#include +#include +#include +#include +#include +#include +#include "hadoopwinutilsvc_h.h" + +#pragma comment(lib, "Rpcrt4.lib") +#pragma comment(lib, "advapi32.lib") +#pragma comment(lib, "authz.lib") + + +#define NM_WSCE_ALLOWED L"yarn.nodemanager.windows-secure-container-executor.allowed" +#define NM_WSCE_DENIED L"yarn.nodemanager.windows-secure-container-executor.denied" + +#define SERVICE_ACCESS_MASK 0x00000001 + +SERVICE_STATUS gSvcStatus; +SERVICE_STATUS_HANDLE gSvcStatusHandle; +HANDLE ghSvcStopEvent = INVALID_HANDLE_VALUE; +HANDLE ghWaitObject = INVALID_HANDLE_VALUE; +HANDLE ghEventLog = INVALID_HANDLE_VALUE; +BOOL isListenning = FALSE; +PSECURITY_DESCRIPTOR pAllowedSD = NULL; + +VOID SvcError(DWORD dwError); +VOID WINAPI SvcMain(DWORD dwArg, LPTSTR* lpszArgv); +DWORD SvcInit(); +DWORD RpcInit(); +DWORD AuthInit(); +VOID ReportSvcStatus( DWORD dwCurrentState, + DWORD dwWin32ExitCode, + DWORD dwWaitHint); +VOID WINAPI SvcCtrlHandler( DWORD dwCtrl ); +VOID CALLBACK SvcShutdown( + _In_ PVOID lpParameter, + _In_ BOOLEAN TimerOrWaitFired); + +#define CHECK_ERROR_DONE(status, expected, category, message) \ + if (status != expected) { \ + ReportSvcCheckError( \ + EVENTLOG_ERROR_TYPE, \ + category, \ + status, \ + message); \ + goto done; \ + } else { \ + LogDebugMessage(L"%s: OK\n", message); \ + } + + +#define CHECK_RPC_STATUS_DONE(status, message) \ + CHECK_ERROR_DONE(status, RPC_S_OK, SERVICE_CATEGORY, message) + +#define CHECK_SVC_STATUS_DONE(status, message) \ + CHECK_ERROR_DONE(status, ERROR_SUCCESS, SERVICE_CATEGORY, message) + +#define CHECK_UNWIND_RPC(rpcCall) { \ + unwindStatus = rpcCall; \ + if (RPC_S_OK != unwindStatus) { \ + ReportSvcCheckError( \ + EVENTLOG_WARNING_TYPE, \ + SERVICE_CATEGORY, \ + unwindStatus, \ + L#rpcCall); \ + } \ + } + + +//---------------------------------------------------------------------------- +// Function: ReportSvcCheckError +// +// Description: +// Reports an error with the system event log and to debugger console (if present) +// +void ReportSvcCheckError(WORD type, WORD category, DWORD dwError, LPCWSTR message) { + int len; + LPWSTR systemMsg = NULL; + LPWSTR appMsg = NULL; + DWORD dwReportError; + LPWSTR reportMsg = NULL; + WCHAR hexError[32]; + LPCWSTR inserts[] = {message, NULL, NULL, NULL}; + HRESULT hr; + + hr = StringCbPrintf(hexError, sizeof(hexError), TEXT("%x"), dwError); + if (SUCCEEDED(hr)) { + inserts[1] = hexError; + } + else { + inserts[1] = L"(Failed to format dwError as string)"; + } + + len = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, dwError, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR)&systemMsg, 0, NULL); + + if (len) { + inserts[2] = systemMsg; + } + else { + inserts[2] = L"(Failed to get the system error message)"; + } + + LogDebugMessage(L"%s:%d %.*s\n", message, dwError, len, systemMsg); + + if (INVALID_HANDLE_VALUE != ghEventLog) { + if (!ReportEvent(ghEventLog, type, category, MSG_CHECK_ERROR, + NULL, // lpUserSid + (WORD) 3, // wNumStrings + (DWORD) 0, // dwDataSize + inserts, // *lpStrings + NULL // lpRawData + )) { + // We tried to report and failed. Send to dbg. + dwReportError = GetLastError(); + len = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, dwReportError, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR)&reportMsg, 0, NULL); + LogDebugMessage(L"ReportEvent: Error:%d %.*s\n", dwReportError, reportMsg); + } + }; + + if (NULL != systemMsg) LocalFree(systemMsg); + if (NULL != reportMsg) LocalFree(reportMsg); +} + + +VOID ReportSvcMessage(WORD type, WORD category, DWORD msgId) { + DWORD dwError; + + if (INVALID_HANDLE_VALUE != ghEventLog) { + if (!ReportEvent(ghEventLog, type, category, msgId, + NULL, // lpUserSid + (WORD) 0, // wNumStrings + (DWORD) 0, // dwDataSize + NULL, // *lpStrings + NULL // lpRawData + )) { + // We tried to report and failed but debugger is attached. Send to dbg. + dwError = GetLastError(); + LogDebugMessage(L"ReportEvent: error %d\n", dwError); + } + } +} + + +//---------------------------------------------------------------------------- +// Function: RunService +// +// Description: +// Registers with NT SCM and starts the service +// +// Returns: +// ERROR_SUCCESS: On success +// Error code otherwise: otherwise +DWORD RunService(__in int argc, __in_ecount(argc) wchar_t *argv[]) +{ + DWORD dwError= ERROR_SUCCESS; + int argStart = 1; + + static const SERVICE_TABLE_ENTRY serviceTable[] = { + { SVCNAME, (LPSERVICE_MAIN_FUNCTION) SvcMain }, + { NULL, NULL } + }; + + dwError = AuthInit(); + if (ERROR_SUCCESS != dwError) { + SvcError(dwError); + goto done; + } + + ghEventLog = RegisterEventSource(NULL, SVCNAME); + if (NULL == ghEventLog) { + dwError = GetLastError(); + CHECK_SVC_STATUS_DONE(dwError, L"RegisterEventSource") + } + + if (!StartServiceCtrlDispatcher(serviceTable)) { + dwError = GetLastError(); + CHECK_SVC_STATUS_DONE(dwError, L"StartServiceCtrlDispatcher") + } + +done: + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: SvcMain +// +// Description: +// Service main entry point. +// +VOID WINAPI SvcMain() { + DWORD dwError = ERROR_SUCCESS; + + gSvcStatusHandle = RegisterServiceCtrlHandler( + SVCNAME, + SvcCtrlHandler); + if( !gSvcStatusHandle ) { + dwError = GetLastError(); + CHECK_SVC_STATUS_DONE(dwError, L"RegisterServiceCtrlHandler") + } + + // These SERVICE_STATUS members remain as set here + gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + gSvcStatus.dwServiceSpecificExitCode = 0; + + // Report initial status to the SCM + ReportSvcStatus( SERVICE_START_PENDING, NO_ERROR, 3000 ); + + // Perform service-specific initialization and work. + dwError = SvcInit(); + +done: + return; +} + +//---------------------------------------------------------------------------- +// Function: SvcInit +// +// Description: +// Initializes the service. +// +DWORD SvcInit() { + DWORD dwError = ERROR_SUCCESS; + + dwError = EnablePrivilege(SE_DEBUG_NAME); + if( dwError != ERROR_SUCCESS ) { + goto done; + } + + // The recommended way to shutdown the service is to use an event + // and attach a callback with RegisterWaitForSingleObject + // + ghSvcStopEvent = CreateEvent( + NULL, // default security attributes + TRUE, // manual reset event + FALSE, // not signaled + NULL); // no name + + if ( ghSvcStopEvent == NULL) + { + dwError = GetLastError(); + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + dwError, L"CreateEvent"); + ReportSvcStatus( SERVICE_STOPPED, dwError, 0 ); + goto done; + } + + if (!RegisterWaitForSingleObject (&ghWaitObject, + ghSvcStopEvent, + SvcShutdown, + NULL, + INFINITE, + WT_EXECUTEONLYONCE)) { + dwError = GetLastError(); + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + dwError, L"RegisterWaitForSingleObject"); + CloseHandle(ghSvcStopEvent); + ReportSvcStatus( SERVICE_STOPPED, dwError, 0 ); + goto done; + } + + // Report running status when initialization is complete. + ReportSvcStatus( SERVICE_RUNNING, NO_ERROR, 0 ); + + dwError = RpcInit(); + +done: + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: RpcAuthorizeCallback +// +// Description: +// RPC Authorization callback. +// +// Returns: +// RPC_S_OK for access authorized +// RPC_S_ACCESS_DENIED for access denied +// +RPC_STATUS CALLBACK RpcAuthorizeCallback ( + RPC_IF_HANDLE hInterface, + void* pContext) +{ + RPC_STATUS status, + unwindStatus, + authStatus = RPC_S_ACCESS_DENIED; + DWORD dwError; + LUID luidReserved2; + AUTHZ_ACCESS_REQUEST request; + AUTHZ_ACCESS_REPLY reply; + AUTHZ_CLIENT_CONTEXT_HANDLE hClientContext = NULL; + DWORD authError = ERROR_SUCCESS; + DWORD saclResult = 0; + ACCESS_MASK grantedMask = 0; + + ZeroMemory(&luidReserved2, sizeof(luidReserved2)); + ZeroMemory(&request, sizeof(request)); + ZeroMemory(&reply, sizeof(reply)); + + status = RpcGetAuthorizationContextForClient(NULL, + FALSE, // ImpersonateOnReturn + NULL, // Reserved1 + NULL, // pExpirationTime + luidReserved2, // Reserved2 + 0, // Reserved3 + NULL, // Reserved4 + &hClientContext); + CHECK_RPC_STATUS_DONE(status, L"RpcGetAuthorizationContextForClient"); + + request.DesiredAccess = MAXIMUM_ALLOWED; + reply.Error = &authError; + reply.SaclEvaluationResults = &saclResult; + reply.ResultListLength = 1; + reply.GrantedAccessMask = &grantedMask; + + if (!AuthzAccessCheck( + 0, + hClientContext, + &request, + NULL, // AuditEvent + pAllowedSD, + NULL, // OptionalSecurityDescriptorArray + 0, // OptionalSecurityDescriptorCount + &reply, + NULL // phAccessCheckResults + )) { + dwError = GetLastError(); + CHECK_SVC_STATUS_DONE(dwError, L"AuthzAccessCheck"); + } + + LogDebugMessage(L"AutzAccessCheck: Error:%d sacl:%d access:%d\n", + authError, saclResult, grantedMask); + if (authError == ERROR_SUCCESS && (grantedMask & SERVICE_ACCESS_MASK)) { + authStatus = RPC_S_OK; + } + +done: + if (NULL != hClientContext) CHECK_UNWIND_RPC(RpcFreeAuthorizationContext(&hClientContext)); + return authStatus; +} + +//---------------------------------------------------------------------------- +// Function: AuthInit +// +// Description: +// Initializes the authorization structures (security descriptor). +// +// Notes: +// This is called from RunService solely for debugging purposed +// so that it can be tested by wimply running winutil service from CLI (no SCM) +// +DWORD AuthInit() { + DWORD dwError = ERROR_SUCCESS; + int count = 0; + int crt = 0; + int len = 0; + LPCWSTR value = NULL; + WCHAR** tokens = NULL; + LPWSTR lpszSD = NULL; + ULONG cchSD = 0; + DWORD dwBufferSize = 0; + int allowedCount = 0; + PSID* allowedSids = NULL; + + + dwError = GetConfigValue( + wsceConfigRelativePath, + NM_WSCE_ALLOWED, &len, &value); + CHECK_SVC_STATUS_DONE(dwError, L"GetConfigValue"); + + if (0 == len) { + CHECK_SVC_STATUS_DONE(ERROR_BAD_CONFIGURATION, NM_WSCE_ALLOWED); + } + + dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens); + CHECK_SVC_STATUS_DONE(dwError, L"SplitStringIgnoreSpaceW"); + + allowedSids = (PSID*) LocalAlloc(LPTR, sizeof(PSID) * count); + for (crt = 0; crt < count; ++crt) { + dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]); + CHECK_SVC_STATUS_DONE(dwError, L"GetSidFromAcctNameW"); + } + + allowedCount = count; + + dwError = BuildServiceSecurityDescriptor(SERVICE_ACCESS_MASK, + allowedCount, allowedSids, 0, NULL, &pAllowedSD); + CHECK_SVC_STATUS_DONE(dwError, L"BuildServiceSecurityDescriptor"); + +done: + if (lpszSD) LocalFree(lpszSD); + if (value) LocalFree(value); + if (tokens) LocalFree(tokens); + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: RpcInit +// +// Description: +// Initializes the RPC infrastructure and starts the RPC listenner. +// +DWORD RpcInit() { + RPC_STATUS status; + DWORD dwError; + + status = RpcServerUseProtseqIf(SVCBINDING, + RPC_C_LISTEN_MAX_CALLS_DEFAULT, + HadoopWinutilSvc_v1_0_s_ifspec, + NULL); + if (RPC_S_OK != status) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + status, L"RpcServerUseProtseqIf"); + SvcError(status); + dwError = status; + goto done; + } + + status = RpcServerRegisterIfEx(HadoopWinutilSvc_v1_0_s_ifspec, + NULL, // MgrTypeUuid + NULL, // MgrEpv + RPC_IF_ALLOW_LOCAL_ONLY, // Flags + RPC_C_LISTEN_MAX_CALLS_DEFAULT, // Max calls + RpcAuthorizeCallback); // Auth callback + + if (RPC_S_OK != status) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + status, L"RpcServerRegisterIfEx"); + SvcError(status); + dwError = status; + goto done; + } + + status = RpcServerListen(1, RPC_C_LISTEN_MAX_CALLS_DEFAULT, TRUE); + if (RPC_S_ALREADY_LISTENING == status) { + ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY, + status, L"RpcServerListen"); + } + else if (RPC_S_OK != status) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + status, L"RpcServerListen"); + SvcError(status); + dwError = status; + goto done; + } + + isListenning = TRUE; + + ReportSvcMessage(EVENTLOG_INFORMATION_TYPE, SERVICE_CATEGORY, + MSG_RPC_SERVICE_HAS_STARTED); + +done: + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: RpcStop +// +// Description: +// Tears down the RPC infrastructure and stops the RPC listenner. +// +VOID RpcStop() { + RPC_STATUS status; + + if (isListenning) { + + status = RpcMgmtStopServerListening(NULL); + isListenning = FALSE; + + if (RPC_S_OK != status) { + ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY, + status, L"RpcMgmtStopServerListening"); + } + + ReportSvcMessage(EVENTLOG_INFORMATION_TYPE, SERVICE_CATEGORY, + MSG_RPC_SERVICE_HAS_STOPPED); + } +} + +//---------------------------------------------------------------------------- +// Function: CleanupHandles +// +// Description: +// Cleans up the global service handles. +// +VOID CleanupHandles() { + if (INVALID_HANDLE_VALUE != ghWaitObject) { + UnregisterWait(ghWaitObject); + ghWaitObject = INVALID_HANDLE_VALUE; + } + if (INVALID_HANDLE_VALUE != ghSvcStopEvent) { + CloseHandle(ghSvcStopEvent); + ghSvcStopEvent = INVALID_HANDLE_VALUE; + } + if (INVALID_HANDLE_VALUE != ghEventLog) { + DeregisterEventSource(ghEventLog); + ghEventLog = INVALID_HANDLE_VALUE; + } +} + +//---------------------------------------------------------------------------- +// Function: SvcError +// +// Description: +// Aborts the startup sequence. Reports error, stops RPC, cleans up globals. +// +VOID SvcError(DWORD dwError) { + RpcStop(); + CleanupHandles(); + ReportSvcStatus( SERVICE_STOPPED, dwError, 0 ); +} + +//---------------------------------------------------------------------------- +// Function: SvcShutdown +// +// Description: +// Callback when the shutdown event is signaled. Stops RPC, cleans up globals. +// +VOID CALLBACK SvcShutdown( + _In_ PVOID lpParameter, + _In_ BOOLEAN TimerOrWaitFired) { + RpcStop(); + CleanupHandles(); + ReportSvcStatus( SERVICE_STOPPED, NO_ERROR, 0 ); +} + +//---------------------------------------------------------------------------- +// Function: SvcCtrlHandler +// +// Description: +// Callback from SCM for for service events (signals). +// +// Notes: +// Shutdown is indirect, we set her the STOP_PENDING state and signal the stop event. +// Signaling the event invokes SvcShutdown which completes the shutdown. +// This two staged approach allows the SCM handler to complete fast, +// not blocking the SCM big fat global lock. +// +VOID WINAPI SvcCtrlHandler( DWORD dwCtrl ) +{ + // Handle the requested control code. + + switch(dwCtrl) + { + case SERVICE_CONTROL_STOP: + ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0); + + // Signal the service to stop. + SetEvent(ghSvcStopEvent); + + return; + + default: + break; + } + +} + +//---------------------------------------------------------------------------- +// Function: ReportSvcStatus +// +// Description: +// Updates the service status with the SCM. +// +VOID ReportSvcStatus( DWORD dwCurrentState, + DWORD dwWin32ExitCode, + DWORD dwWaitHint) +{ + static DWORD dwCheckPoint = 1; + DWORD dwError; + + // Fill in the SERVICE_STATUS structure. + + gSvcStatus.dwCurrentState = dwCurrentState; + gSvcStatus.dwWin32ExitCode = dwWin32ExitCode; + gSvcStatus.dwWaitHint = dwWaitHint; + + if (dwCurrentState == SERVICE_START_PENDING) + gSvcStatus.dwControlsAccepted = 0; + else gSvcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP; + + if ( (dwCurrentState == SERVICE_RUNNING) || + (dwCurrentState == SERVICE_STOPPED) ) + gSvcStatus.dwCheckPoint = 0; + else gSvcStatus.dwCheckPoint = dwCheckPoint++; + + // Report the status of the service to the SCM. + if (!SetServiceStatus( gSvcStatusHandle, &gSvcStatus)) { + dwError = GetLastError(); + ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY, + dwError, L"SetServiceStatus"); + }; +} + +//---------------------------------------------------------------------------- +// Function: WinutilsCreateProcessAsUser +// +// Description: +// The RPC midl declared function implementation +// +// Returns: +// ERROR_SUCCESS: On success +// Error code otherwise: otherwise +// +// Notes: +// This is the entry point when the NodeManager does the RPC call +// Note that the RPC call does not do any S4U work. Is simply spawns (suspended) wintutils +// using the right command line and the handles over the spwaned process to the NM +// The actual S4U work occurs in the spawned process, run and monitored by the NM +// +error_status_t WinutilsCreateProcessAsUser( + /* [in] */ int nmPid, + /* [in] */ CREATE_PROCESS_REQUEST *request, + /* [out] */ CREATE_PROCESS_RESPONSE **response) { + + DWORD dwError = ERROR_SUCCESS; + LPCWSTR inserts[] = {request->cwd, request->jobName, request->user, request->pidFile, request->cmdLine, NULL}; + WCHAR winutilsPath[MAX_PATH]; + WCHAR fullCmdLine[32768]; + HANDLE taskStdInRd = INVALID_HANDLE_VALUE, taskStdInWr = INVALID_HANDLE_VALUE, + taskStdOutRd = INVALID_HANDLE_VALUE, taskStdOutWr = INVALID_HANDLE_VALUE, + taskStdErrRd = INVALID_HANDLE_VALUE, taskStdErrWr = INVALID_HANDLE_VALUE, + hNmProcess = INVALID_HANDLE_VALUE, + hDuplicateProcess = INVALID_HANDLE_VALUE, + hDuplicateThread = INVALID_HANDLE_VALUE, + hDuplicateStdIn = INVALID_HANDLE_VALUE, + hDuplicateStdOut = INVALID_HANDLE_VALUE, + hDuplicateStdErr = INVALID_HANDLE_VALUE, + hSelfProcess = INVALID_HANDLE_VALUE; + BOOL fMustCleanupProcess = FALSE; + + HRESULT hr; + STARTUPINFO si; + PROCESS_INFORMATION pi; + SECURITY_ATTRIBUTES saTaskStdInOutErr; + + ZeroMemory( &si, sizeof(si) ); + si.cb = sizeof(si); + ZeroMemory( &pi, sizeof(pi) ); + pi.hProcess = INVALID_HANDLE_VALUE; + pi.hThread = INVALID_HANDLE_VALUE; + ZeroMemory( &saTaskStdInOutErr, sizeof(saTaskStdInOutErr)); + + // NB: GetCurrentProcess returns a pseudo-handle that just so happens + // has the value -1, ie. INVALID_HANDLE_VALUE. It cannot fail. + // + hSelfProcess = GetCurrentProcess(); + + hNmProcess = OpenProcess(PROCESS_DUP_HANDLE, FALSE, nmPid); + if (NULL == hNmProcess) { + dwError = GetLastError(); + goto done; + } + + GetModuleFileName(NULL, winutilsPath, sizeof(winutilsPath)/sizeof(WCHAR)); + dwError = GetLastError(); // Always check after GetModuleFileName for ERROR_INSSUFICIENT_BUFFER + if (dwError) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + dwError, L"GetModuleFileName"); + goto done; + } + + // NB. We can call CreateProcess("wintuls","task create ...") or we can call + // CreateProcess(NULL, "winutils task create"). Only the second form passes "task" as + // argv[1], as expected by main. First form passes "task" as argv[0] and main fails. + + hr = StringCbPrintf(fullCmdLine, sizeof(fullCmdLine), L"\"%s\" task createAsUser %ls %ls %ls %ls", + winutilsPath, + request->jobName, request->user, request->pidFile, request->cmdLine); + if (FAILED(hr)) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + hr, L"StringCbPrintf:fullCmdLine"); + goto done; + } + + LogDebugMessage(L"[%ls]: %ls %ls\n", request->cwd, winutilsPath, fullCmdLine); + + // stdin/stdout/stderr redirection is handled here + // We create 3 anonimous named pipes. + // Security attributes are required so that the handles can be inherited. + // We assign one end of the pipe to the process (stdin gets a read end, stdout gets a write end) + // We then duplicate the other end in the NM process, and we close our own handle + // Finally we return the duplicate handle values to the NM + // The NM will attach Java file dscriptors to the duplicated handles and + // read/write them as ordinary Java InputStream/OutputStream objects + + si.dwFlags |= STARTF_USESTDHANDLES; + + saTaskStdInOutErr.nLength = sizeof(SECURITY_ATTRIBUTES); + saTaskStdInOutErr.bInheritHandle = TRUE; + saTaskStdInOutErr.lpSecurityDescriptor = NULL; + + if (!CreatePipe(&taskStdInRd, &taskStdInWr, &saTaskStdInOutErr, 0)) { + dwError = GetLastError(); + goto done; + } + if (!SetHandleInformation(taskStdInWr, HANDLE_FLAG_INHERIT, FALSE)) { + dwError = GetLastError(); + goto done; + } + si.hStdInput = taskStdInRd; + + if (!CreatePipe(&taskStdOutRd, &taskStdOutWr, &saTaskStdInOutErr, 0)) { + dwError = GetLastError(); + goto done; + } + if (!SetHandleInformation(taskStdOutRd, HANDLE_FLAG_INHERIT, FALSE)) { + dwError = GetLastError(); + goto done; + } + si.hStdOutput = taskStdOutWr; + + if (!CreatePipe(&taskStdErrRd, &taskStdErrWr, &saTaskStdInOutErr, 0)) { + dwError = GetLastError(); + goto done; + } + if (!SetHandleInformation(taskStdErrRd, HANDLE_FLAG_INHERIT, FALSE)) { + dwError = GetLastError(); + goto done; + } + si.hStdError = taskStdErrWr; + + if (!CreateProcess( + NULL, // lpApplicationName, + fullCmdLine, // lpCommandLine, + NULL, // lpProcessAttributes, + NULL, // lpThreadAttributes, + TRUE, // bInheritHandles, + CREATE_SUSPENDED, // dwCreationFlags, + NULL, // lpEnvironment, + request->cwd, // lpCurrentDirectory, + &si, // lpStartupInfo + &pi)) { // lpProcessInformation + + dwError = GetLastError(); + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + dwError, L"CreateProcess"); + goto done; + } + + fMustCleanupProcess = TRUE; + + LogDebugMessage(L"CreateProcess: pid:%x\n", pi.dwProcessId); + + if (!DuplicateHandle(hSelfProcess, pi.hProcess, hNmProcess, + &hDuplicateProcess, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: pi.hProcess\n"); + goto done; + } + + if (!DuplicateHandle(hSelfProcess, pi.hThread, hNmProcess, + &hDuplicateThread, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: pi.hThread\n"); + goto done; + } + + if (!DuplicateHandle(hSelfProcess, taskStdInWr, hNmProcess, + &hDuplicateStdIn, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: taskStdInWr\n"); + goto done; + } + + if (!DuplicateHandle(hSelfProcess, taskStdOutRd, hNmProcess, + &hDuplicateStdOut, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: taskStdOutRd\n"); + goto done; + } + + if (!DuplicateHandle(hSelfProcess, taskStdErrRd, hNmProcess, + &hDuplicateStdErr, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: taskStdErrRd\n"); + goto done; + } + + *response = (CREATE_PROCESS_RESPONSE*) MIDL_user_allocate(sizeof(CREATE_PROCESS_RESPONSE)); + if (NULL == *response) { + dwError = ERROR_OUTOFMEMORY; + LogDebugMessage(L"Failed to allocate CREATE_PROCESS_RESPONSE* response\n"); + goto done; + } + + // We're now transfering ownership of the duplicated handles to the caller + // If the RPC call fails *after* this point the handles are leaked inside the NM process + + (*response)->hProcess = hDuplicateProcess; + (*response)->hThread = hDuplicateThread; + (*response)->hStdIn = hDuplicateStdIn; + (*response)->hStdOut = hDuplicateStdOut; + (*response)->hStdErr = hDuplicateStdErr; + + fMustCleanupProcess = FALSE; + +done: + + if (fMustCleanupProcess) { + LogDebugMessage(L"Cleaning process: %d due to error:%d\n", pi.dwProcessId, dwError); + TerminateProcess(pi.hProcess, EXIT_FAILURE); + + // cleanup the duplicate handles inside the NM. + + if (INVALID_HANDLE_VALUE != hDuplicateProcess) { + DuplicateHandle(hNmProcess, hDuplicateProcess, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + if (INVALID_HANDLE_VALUE != hDuplicateThread) { + DuplicateHandle(hNmProcess, hDuplicateThread, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + if (INVALID_HANDLE_VALUE != hDuplicateStdIn) { + DuplicateHandle(hNmProcess, hDuplicateStdIn, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + if (INVALID_HANDLE_VALUE != hDuplicateStdOut) { + DuplicateHandle(hNmProcess, hDuplicateStdOut, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + if (INVALID_HANDLE_VALUE != hDuplicateStdErr) { + DuplicateHandle(hNmProcess, hDuplicateStdErr, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + } + + if (INVALID_HANDLE_VALUE != hSelfProcess) CloseHandle(hSelfProcess); + if (INVALID_HANDLE_VALUE != hNmProcess) CloseHandle(hNmProcess); + if (INVALID_HANDLE_VALUE != taskStdInRd) CloseHandle(taskStdInRd); + if (INVALID_HANDLE_VALUE != taskStdInWr) CloseHandle(taskStdInWr); + if (INVALID_HANDLE_VALUE != taskStdOutRd) CloseHandle(taskStdOutRd); + if (INVALID_HANDLE_VALUE != taskStdOutWr) CloseHandle(taskStdOutWr); + if (INVALID_HANDLE_VALUE != taskStdErrRd) CloseHandle(taskStdErrRd); + if (INVALID_HANDLE_VALUE != taskStdErrWr) CloseHandle(taskStdErrWr); + + + // This is closing our own process/thread handles. + // If the transfer was succesfull the NM has its own duplicates (if any) + if (INVALID_HANDLE_VALUE != pi.hThread) CloseHandle(pi.hThread); + if (INVALID_HANDLE_VALUE != pi.hProcess) CloseHandle(pi.hProcess); + + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: ServiceUsage +// +// Description: +// Prints the CLI arguments for service command. +// +void ServiceUsage() +{ + fwprintf(stdout, L"\ + Usage: service\n\ + Starts the nodemanager Windows Secure Container Executor helper service.\n\ + The service must run as a high privileged account (LocalSystem)\n\ + and is used by the nodemanager WSCE to spawn secure containers on Windows.\n"); +} + + diff --git hadoop-common-project/hadoop-common/src/main/winutils/task.c hadoop-common-project/hadoop-common/src/main/winutils/task.c index 783f162..67e82c3 100644 --- hadoop-common-project/hadoop-common/src/main/winutils/task.c +++ hadoop-common-project/hadoop-common/src/main/winutils/task.c @@ -18,29 +18,37 @@ #include "winutils.h" #include #include -#include +#include +#include #define PSAPI_VERSION 1 #pragma comment(lib, "psapi.lib") + +#define NM_WSCE_IMPERSONATE_ALLOWED L"yarn.nodemanager.windows-secure-container-executor.impersonate.allowed" +#define NM_WSCE_IMPERSONATE_DENIED L"yarn.nodemanager.windows-secure-container-executor.impersonate.denied" + +// The S4U impersonation access check mask. Arbitrary value (we use 1 for the service access check) +#define SERVICE_IMPERSONATE_MASK 0x00000002 + #define ERROR_TASK_NOT_ALIVE 1 // This exit code for killed processes is compatible with Unix, where a killed // process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137. #define KILLED_PROCESS_EXIT_CODE 137 -// Name for tracking this logon process when registering with LSA -static const char *LOGON_PROCESS_NAME="Hadoop Container Executor"; -// Name for token source, must be less or eq to TOKEN_SOURCE_LENGTH (currently 8) chars -static const char *TOKEN_SOURCE_NAME = "HadoopEx"; - +// Name for tracking this logon process when registering with LSA +static const char *LOGON_PROCESS_NAME="Hadoop Container Executor"; +// Name for token source, must be less or eq to TOKEN_SOURCE_LENGTH (currently 8) chars +static const char *TOKEN_SOURCE_NAME = "HadoopEx"; + // List of different task related command line options supported by // winutils. typedef enum TaskCommandOptionType { TaskInvalid, TaskCreate, - TaskCreateAsUser, + TaskCreateAsUser, TaskIsAlive, TaskKill, TaskProcessList @@ -93,53 +101,275 @@ static BOOL ParseCommandLine(__in int argc, } } - if (argc >= 6) { - if (wcscmp(argv[1], L"createAsUser") == 0) - { - *command = TaskCreateAsUser; - return TRUE; - } - } - + if (argc >= 6) { + if (wcscmp(argv[1], L"createAsUser") == 0) + { + *command = TaskCreateAsUser; + return TRUE; + } + } + return FALSE; } + //---------------------------------------------------------------------------- -// Function: CreateTaskImpl +// Function: BuildImpersonateSecurityDescriptor +// +// Description: +// Builds the security descriptor for the S4U impersonation permissions +// This describes what users can be impersonated and what not +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +// +DWORD BuildImpersonateSecurityDescriptor(__out PSECURITY_DESCRIPTOR* ppSD) { + DWORD dwError = ERROR_SUCCESS; + size_t countAllowed = 0; + PSID* allowedSids = NULL; + size_t countDenied = 0; + PSID* deniedSids = NULL; + LPCWSTR value = NULL; + WCHAR** tokens = NULL; + size_t len = 0; + size_t count = 0; + int crt = 0; + PSECURITY_DESCRIPTOR pSD = NULL; + + dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_IMPERSONATE_ALLOWED, &len, &value); + if (dwError) { + ReportErrorCode(L"GetConfigValue:1", dwError); + goto done; + } + + if (0 == len) { + dwError = ERROR_BAD_CONFIGURATION; + ReportErrorCode(L"GetConfigValue:2", dwError); + goto done; + } + + dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens); + if (dwError) { + ReportErrorCode(L"SplitStringIgnoreSpaceW:1", dwError); + goto done; + } + + allowedSids = LocalAlloc(LPTR, sizeof(PSID) * count); + if (NULL == allowedSids) { + dwError = GetLastError(); + ReportErrorCode(L"LocalAlloc:1", dwError); + goto done; + } + + for(crt = 0; crt < count; ++crt) { + dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]); + if (dwError) { + ReportErrorCode(L"GetSidFromAcctNameW:1", dwError); + goto done; + } + } + countAllowed = count; + + LocalFree(tokens); + tokens = NULL; + + LocalFree(value); + value = NULL; + + dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_IMPERSONATE_DENIED, &len, &value); + if (dwError) { + ReportErrorCode(L"GetConfigValue:3", dwError); + goto done; + } + + if (0 != len) { + dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens); + if (dwError) { + ReportErrorCode(L"SplitStringIgnoreSpaceW:2", dwError); + goto done; + } + + deniedSids = LocalAlloc(LPTR, sizeof(PSID) * count); + if (NULL == allowedSids) { + dwError = GetLastError(); + ReportErrorCode(L"LocalAlloc:2", dwError); + goto done; + } + + for(crt = 0; crt < count; ++crt) { + dwError = GetSidFromAcctNameW(tokens[crt], &deniedSids[crt]); + if (dwError) { + ReportErrorCode(L"GetSidFromAcctNameW:2", dwError); + goto done; + } + } + countDenied = count; + } + + dwError = BuildServiceSecurityDescriptor( + SERVICE_IMPERSONATE_MASK, + countAllowed, allowedSids, + countDenied, deniedSids, + &pSD); + + if (dwError) { + ReportErrorCode(L"BuildServiceSecurityDescriptor", dwError); + goto done; + } + + *ppSD = pSD; + pSD = NULL; + +done: + if (pSD) LocalFree(pSD); + if (tokens) LocalFree(tokens); + if (allowedSids) LocalFree(allowedSids); + if (deniedSids) LocalFree(deniedSids); + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: ValidateImpersonateAccessCheck +// +// Description: +// Performs the access check for S4U impersonation +// +// Returns: +// ERROR_SUCCESS: On success +// ERROR_ACCESS_DENIED, GetLastError: otherwise +// +DWORD ValidateImpersonateAccessCheck(__in HANDLE logonHandle) { + DWORD dwError = ERROR_SUCCESS; + PSECURITY_DESCRIPTOR pSD = NULL; + LUID luidUnused; + AUTHZ_ACCESS_REQUEST request; + AUTHZ_ACCESS_REPLY reply; + DWORD authError = ERROR_SUCCESS; + DWORD saclResult = 0; + ACCESS_MASK grantedMask = 0; + AUTHZ_RESOURCE_MANAGER_HANDLE hManager = NULL; + AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzToken = NULL; + + ZeroMemory(&luidUnused, sizeof(luidUnused)); + ZeroMemory(&request, sizeof(request)); + ZeroMemory(&reply, sizeof(reply)); + + dwError = BuildImpersonateSecurityDescriptor(&pSD); + if (dwError) { + ReportErrorCode(L"BuildImpersonateSecurityDescriptor", dwError); + goto done; + } + + request.DesiredAccess = MAXIMUM_ALLOWED; + reply.Error = &authError; + reply.SaclEvaluationResults = &saclResult; + reply.ResultListLength = 1; + reply.GrantedAccessMask = &grantedMask; + + if (!AuthzInitializeResourceManager( + AUTHZ_RM_FLAG_NO_AUDIT, + NULL, // pfnAccessCheck + NULL, // pfnComputeDynamicGroups + NULL, // pfnFreeDynamicGroups + NULL, // szResourceManagerName + &hManager)) { + dwError = GetLastError(); + ReportErrorCode(L"AuthzInitializeResourceManager", dwError); + goto done; + } + + if (!AuthzInitializeContextFromToken( + 0, + logonHandle, + hManager, + NULL, // expiration time + luidUnused, // not used + NULL, // callback args + &hAuthzToken)) { + dwError = GetLastError(); + ReportErrorCode(L"AuthzInitializeContextFromToken", dwError); + goto done; + } + + if (!AuthzAccessCheck( + 0, + hAuthzToken, + &request, + NULL, // AuditEvent + pSD, + NULL, // OptionalSecurityDescriptorArray + 0, // OptionalSecurityDescriptorCount + &reply, + NULL // phAccessCheckResults + )) { + dwError = GetLastError(); + ReportErrorCode(L"AuthzAccessCheck", dwError); + goto done; + } + + LogDebugMessage(L"AutzAccessCheck: Error:%d sacl:%d access:%d\n", + authError, saclResult, grantedMask); + + if (authError != ERROR_SUCCESS) { + ReportErrorCode(L"AuthzAccessCheck:REPLY:1", authError); + dwError = authError; + } + else if (!(grantedMask & SERVICE_IMPERSONATE_MASK)) { + ReportErrorCode(L"AuthzAccessCheck:REPLY:2", ERROR_ACCESS_DENIED); + dwError = ERROR_ACCESS_DENIED; + } + +done: + if (hAuthzToken) AuthzFreeContext(hAuthzToken); + if (hManager) AuthzFreeResourceManager(hManager); + if (pSD) LocalFree(pSD); + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: CreateTaskImpl // // Description: // Creates a task via a jobobject. Outputs the // appropriate information to stdout on success, or stderr on failure. -// logonHandle may be NULL, in this case the current logon will be utilized for the -// created process +// logonHandle may be NULL, in this case the current logon will be utilized for the +// created process // // Returns: // ERROR_SUCCESS: On success // GetLastError: otherwise -DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PWSTR cmdLine) +DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PWSTR cmdLine) { - DWORD dwErrorCode = ERROR_SUCCESS; + DWORD dwErrorCode = ERROR_SUCCESS; DWORD exitCode = EXIT_FAILURE; - DWORD currDirCnt = 0; + DWORD currDirCnt = 0; STARTUPINFO si; PROCESS_INFORMATION pi; HANDLE jobObject = NULL; JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = { 0 }; - void * envBlock = NULL; - BOOL createProcessResult = FALSE; - - wchar_t* curr_dir = NULL; - FILE *stream = NULL; + void * envBlock = NULL; + BOOL createProcessResult = FALSE; + + wchar_t* curr_dir = NULL; + FILE *stream = NULL; + + if (NULL != logonHandle) { + dwErrorCode = ValidateImpersonateAccessCheck(logonHandle); + if (dwErrorCode) { + return dwErrorCode; + } + } // Create un-inheritable job object handle and set job object to terminate // when last handle is closed. So winutils.exe invocation has the only open // job object handle. Exit of winutils.exe ensures termination of job object. // Either a clean exit of winutils or crash or external termination. jobObject = CreateJobObject(NULL, jobObjName); - dwErrorCode = GetLastError(); - if(jobObject == NULL || dwErrorCode == ERROR_ALREADY_EXISTS) + dwErrorCode = GetLastError(); + if(jobObject == NULL || dwErrorCode == ERROR_ALREADY_EXISTS) { - return dwErrorCode; + return dwErrorCode; } jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; if(SetInformationJobObject(jobObject, @@ -147,119 +377,121 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW &jeli, sizeof(jeli)) == 0) { - dwErrorCode = GetLastError(); + dwErrorCode = GetLastError(); CloseHandle(jobObject); - return dwErrorCode; + return dwErrorCode; } if(AssignProcessToJobObject(jobObject, GetCurrentProcess()) == 0) { - dwErrorCode = GetLastError(); + dwErrorCode = GetLastError(); CloseHandle(jobObject); - return dwErrorCode; + return dwErrorCode; } // the child JVM uses this env var to send the task OS process identifier // to the TaskTracker. We pass the job object name. if(SetEnvironmentVariable(L"JVM_PID", jobObjName) == 0) { - dwErrorCode = GetLastError(); - // We have to explictly Terminate, passing in the error code - // simply closing the job would kill our own process with success exit status - TerminateJobObject(jobObject, dwErrorCode); - return dwErrorCode; + dwErrorCode = GetLastError(); + // We have to explictly Terminate, passing in the error code + // simply closing the job would kill our own process with success exit status + TerminateJobObject(jobObject, dwErrorCode); + return dwErrorCode; } ZeroMemory( &si, sizeof(si) ); si.cb = sizeof(si); ZeroMemory( &pi, sizeof(pi) ); - if( logonHandle != NULL ) { - // create user environment for this logon - if(!CreateEnvironmentBlock(&envBlock, - logonHandle, - TRUE )) { - dwErrorCode = GetLastError(); - // We have to explictly Terminate, passing in the error code - // simply closing the job would kill our own process with success exit status - TerminateJobObject(jobObject, dwErrorCode); - return dwErrorCode; - } - } - - // Get the required buffer size first - currDirCnt = GetCurrentDirectory(0, NULL); - if (0 < currDirCnt) { - curr_dir = (wchar_t*) alloca(currDirCnt * sizeof(wchar_t)); - assert(curr_dir); - currDirCnt = GetCurrentDirectory(currDirCnt, curr_dir); - } - - if (0 == currDirCnt) { - dwErrorCode = GetLastError(); - // We have to explictly Terminate, passing in the error code - // simply closing the job would kill our own process with success exit status - TerminateJobObject(jobObject, dwErrorCode); - return dwErrorCode; - } - - if (logonHandle == NULL) { - createProcessResult = CreateProcess( - NULL, // ApplicationName - cmdLine, // command line - NULL, // process security attributes - NULL, // thread security attributes - TRUE, // inherit handles - 0, // creation flags - NULL, // environment - curr_dir, // current directory - &si, // startup info - &pi); // process info - } - else { - createProcessResult = CreateProcessAsUser( - logonHandle, // logon token handle - NULL, // Application handle - cmdLine, // command line - NULL, // process security attributes - NULL, // thread security attributes - FALSE, // inherit handles - CREATE_UNICODE_ENVIRONMENT, // creation flags - envBlock, // environment - curr_dir, // current directory - &si, // startup info - &pi); // process info - } - - if (FALSE == createProcessResult) { - dwErrorCode = GetLastError(); - if( envBlock != NULL ) { - DestroyEnvironmentBlock( envBlock ); - envBlock = NULL; - } - // We have to explictly Terminate, passing in the error code - // simply closing the job would kill our own process with success exit status - TerminateJobObject(jobObject, dwErrorCode); - - // This is tehnically dead code, we cannot reach this condition - return dwErrorCode; + if( logonHandle != NULL ) { + // create user environment for this logon + if(!CreateEnvironmentBlock(&envBlock, + logonHandle, + TRUE )) { + dwErrorCode = GetLastError(); + // We have to explictly Terminate, passing in the error code + // simply closing the job would kill our own process with success exit status + TerminateJobObject(jobObject, dwErrorCode); + return dwErrorCode; + } + } + + // Get the required buffer size first + currDirCnt = GetCurrentDirectory(0, NULL); + if (0 < currDirCnt) { + curr_dir = (wchar_t*) alloca(currDirCnt * sizeof(wchar_t)); + assert(curr_dir); + currDirCnt = GetCurrentDirectory(currDirCnt, curr_dir); + } + + if (0 == currDirCnt) { + dwErrorCode = GetLastError(); + // We have to explictly Terminate, passing in the error code + // simply closing the job would kill our own process with success exit status + TerminateJobObject(jobObject, dwErrorCode); + return dwErrorCode; + } + + if (logonHandle == NULL) { + createProcessResult = CreateProcess( + NULL, // ApplicationName + cmdLine, // command line + NULL, // process security attributes + NULL, // thread security attributes + TRUE, // inherit handles + 0, // creation flags + NULL, // environment + curr_dir, // current directory + &si, // startup info + &pi); // process info + } + else { + createProcessResult = CreateProcessAsUser( + logonHandle, // logon token handle + NULL, // Application handle + cmdLine, // command line + NULL, // process security attributes + NULL, // thread security attributes + FALSE, // inherit handles + CREATE_UNICODE_ENVIRONMENT, // creation flags + envBlock, // environment + curr_dir, // current directory + &si, // startup info + &pi); // process info + } + + if (FALSE == createProcessResult) { + dwErrorCode = GetLastError(); + if( envBlock != NULL ) { + DestroyEnvironmentBlock( envBlock ); + envBlock = NULL; + } + // We have to explictly Terminate, passing in the error code + // simply closing the job would kill our own process with success exit status + TerminateJobObject(jobObject, dwErrorCode); + + // This is tehnically dead code, we cannot reach this condition + return dwErrorCode; } CloseHandle(pi.hThread); + ReportErrorCode(L"CreateTaskImpl", ERROR_SUCCESS); + // Wait until child process exits. WaitForSingleObject( pi.hProcess, INFINITE ); if(GetExitCodeProcess(pi.hProcess, &exitCode) == 0) { - dwErrorCode = GetLastError(); + dwErrorCode = GetLastError(); } CloseHandle( pi.hProcess ); - if( envBlock != NULL ) { - DestroyEnvironmentBlock( envBlock ); - envBlock = NULL; - } - + if( envBlock != NULL ) { + DestroyEnvironmentBlock( envBlock ); + envBlock = NULL; + } + // Terminate job object so that all spawned processes are also killed. // This is needed because once this process closes the handle to the job // object and none of the spawned objects have the handle open (via @@ -267,134 +499,148 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW // program (say winutils task kill) to terminate this job object via its name. if(TerminateJobObject(jobObject, exitCode) == 0) { - dwErrorCode = GetLastError(); + dwErrorCode = GetLastError(); } - // comes here only on failure of TerminateJobObject + // comes here only on failure of TerminateJobObject CloseHandle(jobObject); - if(dwErrorCode != ERROR_SUCCESS) + if(dwErrorCode != ERROR_SUCCESS) { - return dwErrorCode; + return dwErrorCode; } return exitCode; } //---------------------------------------------------------------------------- -// Function: CreateTask -// -// Description: -// Creates a task via a jobobject. Outputs the -// appropriate information to stdout on success, or stderr on failure. -// -// Returns: -// ERROR_SUCCESS: On success -// GetLastError: otherwise -DWORD CreateTask(__in PCWSTR jobObjName,__in PWSTR cmdLine) -{ - // call with null logon in order to create tasks utilizing the current logon - return CreateTaskImpl( NULL, jobObjName, cmdLine ); -} -//---------------------------------------------------------------------------- -// Function: CreateTask -// -// Description: -// Creates a task via a jobobject. Outputs the -// appropriate information to stdout on success, or stderr on failure. -// -// Returns: -// ERROR_SUCCESS: On success -// GetLastError: otherwise -DWORD CreateTaskAsUser(__in PCWSTR jobObjName,__in PWSTR user, __in PWSTR pidFilePath, __in PWSTR cmdLine) -{ - DWORD err = ERROR_SUCCESS; - DWORD exitCode = EXIT_FAILURE; - ULONG authnPkgId; - HANDLE lsaHandle = INVALID_HANDLE_VALUE; - PROFILEINFO pi; - BOOL profileIsLoaded = FALSE; - FILE* pidFile = NULL; - - DWORD retLen = 0; - HANDLE logonHandle = NULL; - - err = EnablePrivilege(SE_TCB_NAME); - if( err != ERROR_SUCCESS ) { - fwprintf(stdout, L"INFO: The user does not have SE_TCB_NAME.\n"); - goto done; - } - err = EnablePrivilege(SE_ASSIGNPRIMARYTOKEN_NAME); - if( err != ERROR_SUCCESS ) { - fwprintf(stdout, L"INFO: The user does not have SE_ASSIGNPRIMARYTOKEN_NAME.\n"); - goto done; - } - err = EnablePrivilege(SE_INCREASE_QUOTA_NAME); - if( err != ERROR_SUCCESS ) { - fwprintf(stdout, L"INFO: The user does not have SE_INCREASE_QUOTA_NAME.\n"); - goto done; - } - err = EnablePrivilege(SE_RESTORE_NAME); - if( err != ERROR_SUCCESS ) { - fwprintf(stdout, L"INFO: The user does not have SE_RESTORE_NAME.\n"); - goto done; - } - - err = RegisterWithLsa(LOGON_PROCESS_NAME ,&lsaHandle); - if( err != ERROR_SUCCESS ) goto done; - - err = LookupKerberosAuthenticationPackageId( lsaHandle, &authnPkgId ); - if( err != ERROR_SUCCESS ) goto done; - - err = CreateLogonForUser(lsaHandle, - LOGON_PROCESS_NAME, - TOKEN_SOURCE_NAME, - authnPkgId, - user, - &logonHandle); - if( err != ERROR_SUCCESS ) goto done; - - err = LoadUserProfileForLogon(logonHandle, &pi); - if( err != ERROR_SUCCESS ) goto done; - profileIsLoaded = TRUE; - - // Create the PID file - - if (!(pidFile = _wfopen(pidFilePath, "w"))) { - err = GetLastError(); - goto done; - } - - if (0 > fprintf_s(pidFile, "%ls", jobObjName)) { - err = GetLastError(); - } - - fclose(pidFile); - - if (err != ERROR_SUCCESS) { - goto done; - } - - err = CreateTaskImpl(logonHandle, jobObjName, cmdLine); - -done: - if( profileIsLoaded ) { - UnloadProfileForLogon( logonHandle, &pi ); - profileIsLoaded = FALSE; - } - if( logonHandle != NULL ) { - CloseHandle(logonHandle); - } - - if (INVALID_HANDLE_VALUE != lsaHandle) { - UnregisterWithLsa(lsaHandle); - } - - return err; -} - - -//---------------------------------------------------------------------------- -// Function: IsTaskAlive +// Function: CreateTask +// +// Description: +// Creates a task via a jobobject. Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +DWORD CreateTask(__in PCWSTR jobObjName,__in PWSTR cmdLine) +{ + // call with null logon in order to create tasks utilizing the current logon + return CreateTaskImpl( NULL, jobObjName, cmdLine ); +} +//---------------------------------------------------------------------------- +// Function: CreateTask +// +// Description: +// Creates a task via a jobobject. Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +DWORD CreateTaskAsUser(__in PCWSTR jobObjName, + __in PCWSTR user, __in PCWSTR pidFilePath, __in PCWSTR cmdLine) +{ + DWORD err = ERROR_SUCCESS; + DWORD exitCode = EXIT_FAILURE; + ULONG authnPkgId; + HANDLE lsaHandle = INVALID_HANDLE_VALUE; + PROFILEINFO pi; + BOOL profileIsLoaded = FALSE; + FILE* pidFile = NULL; + + DWORD retLen = 0; + HANDLE logonHandle = NULL; + + err = EnablePrivilege(SE_TCB_NAME); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"EnablePrivilege:SE_TCB_NAME", err); + goto done; + } + err = EnablePrivilege(SE_ASSIGNPRIMARYTOKEN_NAME); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"EnablePrivilege:SE_ASSIGNPRIMARYTOKEN_NAME", err); + goto done; + } + err = EnablePrivilege(SE_INCREASE_QUOTA_NAME); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"EnablePrivilege:SE_INCREASE_QUOTA_NAME", err); + goto done; + } + err = EnablePrivilege(SE_RESTORE_NAME); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"EnablePrivilege:SE_RESTORE_NAME", err); + goto done; + } + + err = RegisterWithLsa(LOGON_PROCESS_NAME ,&lsaHandle); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"RegisterWithLsa", err); + goto done; + } + + err = LookupKerberosAuthenticationPackageId( lsaHandle, &authnPkgId ); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"LookupKerberosAuthenticationPackageId", err); + goto done; + } + + err = CreateLogonForUser(lsaHandle, + LOGON_PROCESS_NAME, + TOKEN_SOURCE_NAME, + authnPkgId, + user, + &logonHandle); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"CreateLogonForUser", err); + goto done; + } + + err = LoadUserProfileForLogon(logonHandle, &pi); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"LoadUserProfileForLogon", err); + goto done; + } + profileIsLoaded = TRUE; + + // Create the PID file + + if (!(pidFile = _wfopen(pidFilePath, "w"))) { + err = GetLastError(); + ReportErrorCode(L"_wfopen:pidFilePath", err); + goto done; + } + + if (0 > fprintf_s(pidFile, "%ls", jobObjName)) { + err = GetLastError(); + } + + fclose(pidFile); + + if (err != ERROR_SUCCESS) { + goto done; + } + + err = CreateTaskImpl(logonHandle, jobObjName, cmdLine); + +done: + if( profileIsLoaded ) { + UnloadProfileForLogon( logonHandle, &pi ); + profileIsLoaded = FALSE; + } + if( logonHandle != NULL ) { + CloseHandle(logonHandle); + } + + if (INVALID_HANDLE_VALUE != lsaHandle) { + UnregisterWithLsa(lsaHandle); + } + + return err; +} + + +//---------------------------------------------------------------------------- +// Function: IsTaskAlive // // Description: // Checks if a task is alive via a jobobject. Outputs the @@ -403,7 +649,7 @@ done: // Returns: // ERROR_SUCCESS: On success // GetLastError: otherwise -DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob) +DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob) { PJOBOBJECT_BASIC_PROCESS_ID_LIST procList; HANDLE jobObject = NULL; @@ -454,7 +700,7 @@ DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob) } //---------------------------------------------------------------------------- -// Function: KillTask +// Function: KillTask // // Description: // Kills a task via a jobobject. Outputs the @@ -463,7 +709,7 @@ DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob) // Returns: // ERROR_SUCCESS: On success // GetLastError: otherwise -DWORD KillTask(PCWSTR jobObjName) +DWORD KillTask(PCWSTR jobObjName) { HANDLE jobObject = OpenJobObject(JOB_OBJECT_TERMINATE, FALSE, jobObjName); if(jobObject == NULL) @@ -487,7 +733,7 @@ DWORD KillTask(PCWSTR jobObjName) } //---------------------------------------------------------------------------- -// Function: PrintTaskProcessList +// Function: PrintTaskProcessList // // Description: // Prints resource usage of all processes in the task jobobject @@ -495,7 +741,7 @@ DWORD KillTask(PCWSTR jobObjName) // Returns: // ERROR_SUCCESS: On success // GetLastError: otherwise -DWORD PrintTaskProcessList(const WCHAR* jobObjName) +DWORD PrintTaskProcessList(const WCHAR* jobObjName) { DWORD i; PJOBOBJECT_BASIC_PROCESS_ID_LIST procList; @@ -579,21 +825,21 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) { DWORD dwErrorCode = ERROR_SUCCESS; TaskCommandOption command = TaskInvalid; - wchar_t* cmdLine = NULL; - wchar_t buffer[16*1024] = L""; // 32K max command line - size_t charCountBufferLeft = sizeof (buffer)/sizeof(wchar_t); - int crtArgIndex = 0; - size_t argLen = 0; - size_t wscatErr = 0; - wchar_t* insertHere = NULL; - - enum { - ARGC_JOBOBJECTNAME = 2, - ARGC_USERNAME, - ARGC_PIDFILE, - ARGC_COMMAND, - ARGC_COMMAND_ARGS - }; + wchar_t* cmdLine = NULL; + wchar_t buffer[16*1024] = L""; // 32K max command line + size_t charCountBufferLeft = sizeof(buffer)/sizeof(wchar_t); + int crtArgIndex = 0; + size_t argLen = 0; + size_t wscatErr = 0; + wchar_t* insertHere = NULL; + + enum { + ARGC_JOBOBJECTNAME = 2, + ARGC_USERNAME, + ARGC_PIDFILE, + ARGC_COMMAND, + ARGC_COMMAND_ARGS + }; if (!ParseCommandLine(argc, argv, &command)) { dwErrorCode = ERROR_INVALID_COMMAND_LINE; @@ -607,57 +853,57 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) { // Create the task jobobject // - dwErrorCode = CreateTask(argv[2], argv[3]); - if (dwErrorCode != ERROR_SUCCESS) - { - ReportErrorCode(L"CreateTask", dwErrorCode); - goto TaskExit; - } - } else if (command == TaskCreateAsUser) - { - // Create the task jobobject as a domain user - // createAsUser accepts an open list of arguments. All arguments after the command are - // to be passed as argumrnts to the command itself.Here we're concatenating all - // arguments after the command into a single arg entry. - // - cmdLine = argv[ARGC_COMMAND]; - if (argc > ARGC_COMMAND_ARGS) { - crtArgIndex = ARGC_COMMAND; - insertHere = buffer; - while (crtArgIndex < argc) { - argLen = wcslen(argv[crtArgIndex]); - wscatErr = wcscat_s(insertHere, charCountBufferLeft, argv[crtArgIndex]); - switch (wscatErr) { - case 0: - // 0 means success; - break; - case EINVAL: - dwErrorCode = ERROR_INVALID_PARAMETER; - goto TaskExit; - case ERANGE: - dwErrorCode = ERROR_INSUFFICIENT_BUFFER; - goto TaskExit; - default: - // This case is not MSDN documented. - dwErrorCode = ERROR_GEN_FAILURE; - goto TaskExit; - } - insertHere += argLen; - charCountBufferLeft -= argLen; - insertHere[0] = L' '; - insertHere += 1; - charCountBufferLeft -= 1; - insertHere[0] = 0; - ++crtArgIndex; - } - cmdLine = buffer; - } - - dwErrorCode = CreateTaskAsUser( - argv[ARGC_JOBOBJECTNAME], argv[ARGC_USERNAME], argv[ARGC_PIDFILE], cmdLine); + dwErrorCode = CreateTask(argv[2], argv[3]); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"CreateTask", dwErrorCode); + goto TaskExit; + } + } else if (command == TaskCreateAsUser) + { + // Create the task jobobject as a domain user + // createAsUser accepts an open list of arguments. All arguments after the command are + // to be passed as argumrnts to the command itself.Here we're concatenating all + // arguments after the command into a single arg entry. + // + cmdLine = argv[ARGC_COMMAND]; + if (argc > ARGC_COMMAND_ARGS) { + crtArgIndex = ARGC_COMMAND; + insertHere = buffer; + while (crtArgIndex < argc) { + argLen = wcslen(argv[crtArgIndex]); + wscatErr = wcscat_s(insertHere, charCountBufferLeft, argv[crtArgIndex]); + switch (wscatErr) { + case 0: + // 0 means success; + break; + case EINVAL: + dwErrorCode = ERROR_INVALID_PARAMETER; + goto TaskExit; + case ERANGE: + dwErrorCode = ERROR_INSUFFICIENT_BUFFER; + goto TaskExit; + default: + // This case is not MSDN documented. + dwErrorCode = ERROR_GEN_FAILURE; + goto TaskExit; + } + insertHere += argLen; + charCountBufferLeft -= argLen; + insertHere[0] = L' '; + insertHere += 1; + charCountBufferLeft -= 1; + insertHere[0] = 0; + ++crtArgIndex; + } + cmdLine = buffer; + } + + dwErrorCode = CreateTaskAsUser( + argv[ARGC_JOBOBJECTNAME], argv[ARGC_USERNAME], argv[ARGC_PIDFILE], cmdLine); if (dwErrorCode != ERROR_SUCCESS) { - ReportErrorCode(L"CreateTaskAsUser", dwErrorCode); + ReportErrorCode(L"CreateTaskAsUser", dwErrorCode); goto TaskExit; } } else if (command == TaskIsAlive) @@ -666,10 +912,10 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) // int isAlive; int numProcs; - dwErrorCode = IsTaskAlive(argv[2], &isAlive, &numProcs); + dwErrorCode = IsTaskAlive(argv[2], &isAlive, &numProcs); if (dwErrorCode != ERROR_SUCCESS) { - ReportErrorCode(L"IsTaskAlive", dwErrorCode); + ReportErrorCode(L"IsTaskAlive", dwErrorCode); goto TaskExit; } @@ -681,27 +927,27 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) else { dwErrorCode = ERROR_TASK_NOT_ALIVE; - ReportErrorCode(L"IsTaskAlive returned false", dwErrorCode); + ReportErrorCode(L"IsTaskAlive returned false", dwErrorCode); goto TaskExit; } } else if (command == TaskKill) { // Check if task jobobject // - dwErrorCode = KillTask(argv[2]); + dwErrorCode = KillTask(argv[2]); if (dwErrorCode != ERROR_SUCCESS) { - ReportErrorCode(L"KillTask", dwErrorCode); + ReportErrorCode(L"KillTask", dwErrorCode); goto TaskExit; } } else if (command == TaskProcessList) { // Check if task jobobject // - dwErrorCode = PrintTaskProcessList(argv[2]); + dwErrorCode = PrintTaskProcessList(argv[2]); if (dwErrorCode != ERROR_SUCCESS) { - ReportErrorCode(L"PrintTaskProcessList", dwErrorCode); + ReportErrorCode(L"PrintTaskProcessList", dwErrorCode); goto TaskExit; } } else @@ -712,6 +958,7 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) } TaskExit: + ReportErrorCode(L"TaskExit:", dwErrorCode); return dwErrorCode; } @@ -722,12 +969,12 @@ void TaskUsage() // ProcessTree.isSetsidSupported() fwprintf(stdout, L"\ Usage: task create [TASKNAME] [COMMAND_LINE] |\n\ - task createAsUser [TASKNAME] [USERNAME] [PIDFILE] [COMMAND_LINE] |\n\ + task createAsUser [TASKNAME] [USERNAME] [PIDFILE] [COMMAND_LINE] |\n\ task isAlive [TASKNAME] |\n\ task kill [TASKNAME]\n\ task processList [TASKNAME]\n\ Creates a new task jobobject with taskname\n\ - Creates a new task jobobject with taskname as the user provided\n\ + Creates a new task jobobject with taskname as the user provided\n\ Checks if task jobobject is alive\n\ Kills task jobobject\n\ Prints to stdout a list of processes in the task\n\ diff --git hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc new file mode 100644 index 0000000..a2e30ad --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc @@ -0,0 +1,46 @@ +; // winutils.mc + +; // EventLog messages for Hadoop winutils service. + + +LanguageNames=(English=0x409:MSG00409) + + +; // The following are the categories of events. + +MessageIdTypedef=WORD + +MessageId=0x1 +SymbolicName=SERVICE_CATEGORY +Language=English +Service Events +. + +MessageId=0x2 +SymbolicName=LOG_CATEGORY +Language=English +Task Events +. + +; // The following are the message definitions. + +MessageIdTypedef=DWORD + +MessageId=0x80 +SymbolicName=MSG_CHECK_ERROR +Language=English +%1. Error %2: %3. +. + +MessageId=0x100 +SymbolicName=MSG_RPC_SERVICE_HAS_STARTED +Language=English +The LPC server is listenning. +. + +MessageId=0x200 +SymbolicName=MSG_RPC_SERVICE_HAS_STOPPED +Language=English +The LPC server has stopped listenning. +. + diff --git hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln index d4e019e..e2a36be 100644 --- hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln +++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln @@ -1,22 +1,6 @@  Microsoft Visual Studio Solution File, Format Version 11.00 # Visual Studio 2010 - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winutils", "winutils.vcxproj", "{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}" ProjectSection(ProjectDependencies) = postProject {12131AA7-902E-4A6D-9CE3-043261D22A12} = {12131AA7-902E-4A6D-9CE3-043261D22A12} @@ -26,26 +10,16 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwinutils", "libwinutils. EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 - Release|Win32 = Release|Win32 Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.ActiveCfg = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.Build.0 = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.ActiveCfg = Release|Win32 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.Build.0 = Release|Win32 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Release|x64 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Release|x64 {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.ActiveCfg = Release|x64 {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.Build.0 = Release|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.ActiveCfg = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.Build.0 = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.ActiveCfg = Release|Win32 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.Build.0 = Release|Win32 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Release|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Release|x64 {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.ActiveCfg = Release|x64 {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.Build.0 = Release|x64 EndGlobalSection diff --git hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj index 5b9a195..3b9d79a 100644 --- hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj +++ hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj @@ -1,5 +1,4 @@  - - - - Debug - Win32 - Debug x64 - - Release - Win32 - Release x64 @@ -42,22 +32,11 @@ winutils - - Application - true - Unicode - Application true Unicode - - Application - false - true - Unicode - Application false @@ -67,15 +46,9 @@ - - - - - - @@ -83,57 +56,32 @@ include;$(IncludePath) - - true - - - true - - ..\..\..\target\winutils\$(Configuration)\ - - - false - false - ..\..\..\target\winutils\$(Platform)\$(Configuration)\ - ..\..\..\target\bin\ - - - - - Level3 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - - - Console - true - - Level4 Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + WIN32;_CONSOLE;_DEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions) Console true - + Level3 - MaxSpeed + + Disabled true true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + WIN32;_CONSOLE;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions) Console @@ -142,24 +90,40 @@ true - + - Level3 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + $(IntermediateOutputPath) - - Console - true - true - true - + + Compiling Messages + mc.exe $(TargetName).mc -z $(TargetName)_msg -r $(IntermediateOutputPath) -h $(IntermediateOutputPath) -U + $(IntermediateOutputPath)$(TargetName)_msg.rc;$(IntermediateOutputPath)$(TargetName)_msg.h + + + true + X64 + $(IntermediateOutputPath) + true + true + true + 2 + + + Midl + ClCompile,ResourceCompile + + + + + + + + + + + @@ -179,4 +143,4 @@ - \ No newline at end of file + diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java index 2f8b84d..1e2d16e 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java @@ -296,7 +296,7 @@ public static boolean isAlive(String pid) { return false; } catch (IOException ioe) { LOG.warn("Error executing shell command " - + Arrays.toString(shexec.getExecString()) + ioe); + + shexec.toString() + ioe); return false; } return (shexec.getExitCode() == 0 ? true : false); @@ -321,7 +321,7 @@ public static boolean isProcessGroupAlive(String pgrpId) { return false; } catch (IOException ioe) { LOG.warn("Error executing shell command " - + Arrays.toString(shexec.getExecString()) + ioe); + + shexec.toString() + ioe); return false; } return (shexec.getExitCode() == 0 ? true : false); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 09897fc..9ece735 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -887,7 +887,7 @@ */ public static final String NM_WINDOWS_SECURE_CONTAINER_GROUP = NM_PREFIX + "windows-secure-container-executor.group"; - + /** T-file compression types used to compress aggregated logs.*/ public static final String NM_LOG_AGG_COMPRESSION_TYPE = NM_PREFIX + "log-aggregation.compression-type"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 23d2e72..ce66c90 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -31,9 +31,11 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; @@ -41,6 +43,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; +import org.apache.hadoop.util.Shell.ICommandExecutor; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -183,20 +186,16 @@ public int launchContainer(Container container, // create log dir under app // fork script - ShellCommandExecutor shExec = null; + Shell.ICommandExecutor shExec = null; try { setScriptExecutable(launchDst, userName); setScriptExecutable(sb.getWrapperScriptPath(), userName); - // Setup command to run - String[] command = getRunCommand(sb.getWrapperScriptPath().toString(), - containerIdStr, userName, pidFile, this.getConf()); - - LOG.info("launchContainer: " + Arrays.toString(command)); - shExec = new ShellCommandExecutor( - command, + shExec = buildCommandExecutor(sb.getWrapperScriptPath().toString(), + containerIdStr, userName, pidFile, this.getConf(), new File(containerWorkDir.toUri().getPath()), - container.getLaunchContext().getEnvironment()); // sanitized env + container.getLaunchContext().getEnvironment()); + if (isContainerActive(containerId)) { shExec.execute(); } @@ -241,11 +240,25 @@ public int launchContainer(Container container, } return exitCode; } finally { - ; // + shExec.dispose(); // } return 0; } + protected ICommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr, + String userName, Path pidFile, Configuration conf, File wordDir, Map environment) + throws IOException { + + String[] command = getRunCommand(wrapperScriptPath, + containerIdStr, userName, pidFile, this.getConf()); + + LOG.info("launchContainer: " + Arrays.toString(command)); + return new ShellCommandExecutor( + command, + wordDir, + environment); + } + protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder( String containerIdStr, Path containerWorkDir) { return Shell.WINDOWS ? diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index 30beaf8..baa0b58 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -17,25 +17,34 @@ */ package org.apache.hadoop.yarn.server.nodemanager; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.PrintStream; +import java.io.Reader; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.io.nativeio.NativeIO.WinutilsProcessStub; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.Shell.ICommandExecutor; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.LocalWrapperScriptBuilder; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; /** @@ -43,7 +52,7 @@ * */ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor { - + private static final Log LOG = LogFactory .getLog(WindowsSecureContainerExecutor.class); @@ -59,6 +68,125 @@ protected void writeLocalWrapperScript(Path launchDst, Path pidFile, PrintStream pout.format("@call \"%s\"", launchDst); } } + + private static class WintuilsProcessStubExecutor implements Shell.ICommandExecutor { + private WinutilsProcessStub processStub; + private StringBuilder output = new StringBuilder(); + private int exitCode; + + private enum State { + INIT, + RUNNING, + COMPLETE + }; + + private State state;; + + private final String cwd; + private final String jobName; + private final String userName; + private final String pidFile; + private final String cmdLine; + private final Configuration conf; + + public WintuilsProcessStubExecutor( + Configuration conf, + String cwd, + String jobName, + String userName, + String pidFile, + String cmdLine) { + this.conf = conf; + this.cwd = cwd; + this.jobName = jobName; + this.userName = userName; + this.pidFile = pidFile; + this.cmdLine = cmdLine; + this.state = State.INIT; + } + + private void assertComplete() throws IOException { + if (state != State.COMPLETE) { + throw new IOException("Process is not complete"); + } + } + + public String getOutput () throws IOException { + assertComplete(); + return output.toString(); + } + + public int getExitCode() throws IOException { + assertComplete(); + return exitCode; + } + + public void validateResult() throws IOException { + assertComplete(); + if (0 != exitCode) { + LOG.warn(output.toString()); + throw new IOException("Processs exit code is:" + exitCode); + } + } + + private Thread startStreamReader(final InputStream stream) throws IOException { + Thread streamReaderThread = new Thread() { + + @Override + public void run() { + try + { + BufferedReader rdr = new BufferedReader( + new InputStreamReader(stream)); + String line = rdr.readLine(); + while((line != null) && !isInterrupted()) { + synchronized(output) { + output.append(line); + output.append(System.getProperty("line.separator")); + } + line = rdr.readLine(); + } + } + catch(Throwable t) { + LOG.error("Error occured reading the process stdout", t); + } + } + }; + streamReaderThread.start(); + return streamReaderThread; + } + + public void execute() throws IOException { + if (state != State.INIT) { + throw new IOException("Process is already started"); + } + processStub = NativeIO.createTaskAsUser(cwd, jobName, userName, pidFile, cmdLine); + state = State.RUNNING; + + Thread stdOutReader = startStreamReader(processStub.getInputStream()); + Thread stdErrReader = startStreamReader(processStub.getErrorStream()); + + try { + processStub.resume(); + processStub.waitFor(); + stdOutReader.join(); + stdErrReader.join(); + } + catch(InterruptedException ie) { + throw new IOException(ie); + } + + exitCode = processStub.exitValue(); + state = State.COMPLETE; + } + + @Override + public void dispose() { + if (processStub != null) { + processStub.dispose(); + } + } + } private String nodeManagerGroup; @@ -125,21 +253,12 @@ public void startLocalizer(Path nmPrivateContainerTokens, copyFile(nmPrivateContainerTokens, tokenDst, user); List command ; - String[] commandArray; - ShellCommandExecutor shExec; File cwdApp = new File(appStorageDir.toString()); LOG.info(String.format("cwdApp: %s", cwdApp)); command = new ArrayList(); - command.add(Shell.WINUTILS); - command.add("task"); - command.add("createAsUser"); - command.add("START_LOCALIZER_" + locId); - command.add(user); - command.add("nul:"); // PID file - //use same jvm as parent File jvm = new File(new File(System.getProperty("java.home"), "bin"), "java.exe"); command.add(jvm.toString()); @@ -160,12 +279,31 @@ public void startLocalizer(Path nmPrivateContainerTokens, } ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, localDirs); - commandArray = command.toArray(new String[command.size()]); - - shExec = new ShellCommandExecutor( - commandArray, cwdApp); - - shExec.execute(); + + String cmdLine = StringUtils.join(command, " "); + + WintuilsProcessStubExecutor stubExecutor = new WintuilsProcessStubExecutor( + getConf(), + cwdApp.getAbsolutePath(), + "START_LOCALIZER_" + locId, user, "nul:", cmdLine); + try { + stubExecutor.execute(); + stubExecutor.validateResult(); + } + finally { + stubExecutor.dispose(); + } } + + @Override + protected ICommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr, + String userName, Path pidFile, Configuration conf, File wordDir, Map environment) + throws IOException { + + return new WintuilsProcessStubExecutor( + getConf(), + wordDir.toString(), + containerIdStr, userName, pidFile.toString(), "cmd /c " + wrapperScriptPath); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 3525170..762565b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -369,10 +369,13 @@ public static void main(String[] argv) throws Throwable { new ContainerLocalizer(FileContext.getLocalFSFileContext(), user, appId, locId, localDirs, RecordFactoryProvider.getRecordFactory(null)); - System.exit(localizer.runLocalization(nmAddr)); + int nRet = localizer.runLocalization(nmAddr); + LOG.info(String.format("nRet: %d", nRet)); + System.exit(nRet); } catch (Throwable e) { // Print error to stdout so that LCE can use it. e.printStackTrace(System.out); + LOG.error("Exception in main:", e); throw e; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java index fc88f94..f6f0e9f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java @@ -277,7 +277,7 @@ public Object answer(InvocationOnMock invocationOnMock) mockExec.createUserLocalDirs(localDirs, appSubmitter); mockExec.createUserCacheDirs(localDirs, appSubmitter); mockExec.createAppDirs(localDirs, appSubmitter, appId); - mockExec.createAppLogDirs(appId, logDirs); + mockExec.createAppLogDirs(appId, logDirs, appSubmitter); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null");