diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index f8e9edf..e9bf88c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -567,6 +567,21 @@ public static boolean access(String path, AccessRight desiredAccess) * @see POSIX#mlock(ByteBuffer, long) */ public static native void extendWorkingSetSize(long delta) throws IOException; + + public static FileDescriptor getFileDescriptorFromHandle(long handle) throws IOException { + if (!Shell.WINDOWS) { + throw new IOException("NativeIO.getFileDescriptorFromHandle only works on Windows"); + } + else if (!nativeLoaded) { + throw new IOException("NativeIO libraries are required for getFileDescriptorFromHandle"); + } + else { + return getFileDescriptorFromHandle0(handle); + } + } + + public static native FileDescriptor getFileDescriptorFromHandle0(long handle); + static { if (NativeCodeLoader.isNativeCodeLoaded()) { @@ -848,71 +863,4 @@ private static native void renameTo0(String src, String dst) private static native void link0(String src, String dst) throws NativeIOException; - - /** - * Wraps a process started by the winutils service helper. - * - */ - public static class WinutilsProcessStub extends Process { - - private final long hProcess; - private final long hThread; - private boolean disposed = false; - - private final InputStream stdErr; - private final InputStream stdOut; - private final OutputStream stdIn; - - public WinutilsProcessStub(long hProcess, long hThread, long hStdIn, long hStdOut, long hStdErr) { - this.hProcess = hProcess; - this.hThread = hThread; - - this.stdIn = new FileOutputStream(getFileDescriptorFromHandle(hStdIn)); - this.stdOut = new FileInputStream(getFileDescriptorFromHandle(hStdOut)); - this.stdErr = new FileInputStream(getFileDescriptorFromHandle(hStdErr)); - } - - private static native FileDescriptor getFileDescriptorFromHandle(long handle); - - @Override - public native void destroy(); - - @Override - public native int exitValue(); - - @Override - public InputStream getErrorStream() { - return stdErr; - } - @Override - public InputStream getInputStream() { - return stdOut; - } - @Override - public OutputStream getOutputStream() { - return stdIn; - } - @Override - public native int waitFor() throws InterruptedException; - - public synchronized native void dispose(); - - public native void resume() throws NativeIOException; - } - - public synchronized static WinutilsProcessStub createTaskAsUser( - String cwd, String jobName, String user, String pidFile, String cmdLine) - throws IOException { - if (!nativeLoaded) { - throw new IOException("NativeIO libraries are required for createTaskAsUser"); - } - synchronized(Shell.WindowsProcessLaunchLock) { - return createTaskAsUser0(cwd, jobName, user, pidFile, cmdLine); - } - } - - private static native WinutilsProcessStub createTaskAsUser0( - String cwd, String jobName, String user, String pidFile, String cmdLine) - throws NativeIOException; - } diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj index e743788..0d67e1e 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj +++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj @@ -99,7 +99,6 @@ - diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index bcf3608..0746de2 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -49,7 +49,6 @@ #include "file_descriptor.h" #include "errno_enum.h" -#include "winutils_process_stub.h" #define MMAP_PROT_READ org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_READ #define MMAP_PROT_WRITE org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_WRITE @@ -231,11 +230,6 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_initNative( PASS_EXCEPTIONS_GOTO(env, error); #endif -#ifdef WINDOWS - winutils_process_stub_init(env); - PASS_EXCEPTIONS_GOTO(env, error); -#endif - return; error: // these are all idempodent and safe to call even if the @@ -248,9 +242,6 @@ error: #ifdef UNIX errno_enum_deinit(env); #endif -#ifdef WINDOWS - winutils_process_stub_deinit(env); -#endif } /* @@ -1157,82 +1148,15 @@ JNIEnv *env, jclass clazz) #endif } - /* - * Class: org_apache_hadoop_io_nativeio_NativeIO - * Method: createTaskAsUser - * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String)Lorg/apache/hadoop/io/nativeio/NativeIO$WinutilsProcessStub + * native static FileDescriptor getFileDescriptorFromHandle0(long handle); + * */ -JNIEXPORT jobject JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_createTaskAsUser0(JNIEnv* env, - jclass clazz, jstring cwd, jstring jobName, jstring user, jstring pidFile, jstring cmdLine) { -#ifdef UNIX - THROW(env, "java/io/IOException", - "The function createTaskAsUser is not supported on Unix"); - return -1; -#endif - -#ifdef WINDOWS - LPCWSTR lpszCwd = NULL, lpszJobName = NULL, - lpszUser = NULL, lpszPidFile = NULL, lpszCmdLine = NULL; - DWORD dwError = ERROR_SUCCESS; - HANDLE hProcess = INVALID_HANDLE_VALUE, - hThread = INVALID_HANDLE_VALUE, - hStdIn = INVALID_HANDLE_VALUE, - hStdOut = INVALID_HANDLE_VALUE, - hStdErr = INVALID_HANDLE_VALUE; - jobject ret = NULL; - - lpszCwd = (LPCWSTR) (*env)->GetStringChars(env, cwd, NULL); - if (!lpszCwd) goto done; // exception was thrown - - lpszJobName = (LPCWSTR) (*env)->GetStringChars(env, jobName, NULL); - if (!lpszJobName) goto done; // exception was thrown +JNIEXPORT jobject JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_getFileDescriptorFromHandle0( + JNIEnv *env, jclass klass, jlong handle) { - lpszUser = (LPCWSTR) (*env)->GetStringChars(env, user, NULL); - if (!lpszUser) goto done; // exception was thrown - - lpszPidFile = (LPCWSTR) (*env)->GetStringChars(env, pidFile, NULL); - if (!lpszPidFile) goto done; // exception was thrown - - lpszCmdLine = (LPCWSTR) (*env)->GetStringChars(env, cmdLine, NULL); - if (!lpszCmdLine) goto done; // exception was thrown - - LogDebugMessage(L"createTaskAsUser: cwd:%s job:%s user:%s pid:%s cmd:%s\n", - lpszCwd, lpszJobName, lpszUser, lpszPidFile, lpszCmdLine); - - dwError = RpcCall_TaskCreateAsUser(lpszCwd, lpszJobName, lpszUser, lpszPidFile, lpszCmdLine, - &hProcess, &hThread, &hStdIn, &hStdOut, &hStdErr); - - if (ERROR_SUCCESS == dwError) { - ret = winutils_process_stub_create(env, (jlong) hProcess, (jlong) hThread, - (jlong) hStdIn, (jlong) hStdOut, (jlong) hStdErr); - - if (NULL == ret) { - TerminateProcess(hProcess, EXIT_FAILURE); - CloseHandle(hThread); - CloseHandle(hProcess); - CloseHandle(hStdIn); - CloseHandle(hStdOut); - CloseHandle(hStdErr); - } - } - -done: - - if (lpszCwd) (*env)->ReleaseStringChars(env, cwd, lpszCwd); - if (lpszJobName) (*env)->ReleaseStringChars(env, jobName, lpszJobName); - if (lpszUser) (*env)->ReleaseStringChars(env, user, lpszUser); - if (lpszPidFile) (*env)->ReleaseStringChars(env, pidFile, lpszPidFile); - if (lpszCmdLine) (*env)->ReleaseStringChars(env, cmdLine, lpszCmdLine); - - if (dwError != ERROR_SUCCESS) { - throw_ioe (env, dwError); - } - - return ret; - -#endif + return fd_create(env, (long) handle); } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.c deleted file mode 100644 index d8afcca..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.c +++ /dev/null @@ -1,189 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with this -* work for additional information regarding copyright ownership. The ASF -* licenses this file to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -* License for the specific language governing permissions and limitations under -* the License. -*/ - -#include -#include "org_apache_hadoop.h" -#include "winutils_process_stub.h" -#include "winutils.h" -#include "file_descriptor.h" - -// class of org.apache.hadoop.io.nativeio.NativeIO.WinutilsProcessStub -static jclass wps_class = NULL; - - -static jmethodID wps_constructor = NULL; -static jfieldID wps_hProcess = NULL; -static jfieldID wps_hThread = NULL; -static jfieldID wps_disposed = NULL; - -extern void throw_ioe(JNIEnv* env, int errnum); - -void winutils_process_stub_init(JNIEnv *env) { - if (wps_class != NULL) return; // already initted - - wps_class = (*env)->FindClass(env, WINUTILS_PROCESS_STUB_CLASS); - PASS_EXCEPTIONS(env); - wps_class = (*env)->NewGlobalRef(env, wps_class); - - wps_hProcess = (*env)->GetFieldID(env, wps_class, "hProcess", "J"); - PASS_EXCEPTIONS(env); - - wps_hThread = (*env)->GetFieldID(env, wps_class, "hThread", "J"); - PASS_EXCEPTIONS(env); - - wps_disposed = (*env)->GetFieldID(env, wps_class, "disposed", "Z"); - PASS_EXCEPTIONS(env); - - wps_constructor = (*env)->GetMethodID(env, wps_class, "", "(JJJJJ)V"); - - LogDebugMessage(L"winutils_process_stub_init\n"); -} - -void winutils_process_stub_deinit(JNIEnv *env) { - if (wps_class != NULL) { - (*env)->DeleteGlobalRef(env, wps_class); - wps_class = NULL; - } - wps_hProcess = NULL; - wps_hThread = NULL; - wps_disposed = NULL; - wps_constructor = NULL; - LogDebugMessage(L"winutils_process_stub_deinit\n"); -} - -jobject winutils_process_stub_create(JNIEnv *env, - jlong hProcess, jlong hThread, jlong hStdIn, jlong hStdOut, jlong hStdErr) { - jobject obj = (*env)->NewObject(env, wps_class, wps_constructor, - hProcess, hThread, hStdIn, hStdOut, hStdErr); - PASS_EXCEPTIONS_RET(env, NULL); - - LogDebugMessage(L"winutils_process_stub_create: %p\n", obj); - - return obj; -} - - -/* - * native void destroy(); - * - * The "00024" in the function name is an artifact of how JNI encodes - * special characters. U+0024 is '$'. - */ -JNIEXPORT void JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_destroy( - JNIEnv *env, jobject objSelf) { - - HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); - LogDebugMessage(L"TerminateProcess: %x\n", hProcess); - TerminateProcess(hProcess, EXIT_FAILURE); -} - -/* - * native void waitFor(); - * - * The "00024" in the function name is an artifact of how JNI encodes - * special characters. U+0024 is '$'. - */ -JNIEXPORT void JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_waitFor( - JNIEnv *env, jobject objSelf) { - - HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); - LogDebugMessage(L"WaitForSingleObject: %x\n", hProcess); - WaitForSingleObject(hProcess, INFINITE); -} - - - -/* - * native void resume(); - * - * The "00024" in the function name is an artifact of how JNI encodes - * special characters. U+0024 is '$'. - */ -JNIEXPORT void JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_resume( - JNIEnv *env, jobject objSelf) { - - DWORD dwError; - HANDLE hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread); - if (-1 == ResumeThread(hThread)) { - dwError = GetLastError(); - LogDebugMessage(L"ResumeThread: %x error:%d\n", hThread, dwError); - throw_ioe(env, dwError); - } -} - -/* - * native int exitValue(); - * - * The "00024" in the function name is an artifact of how JNI encodes - * special characters. U+0024 is '$'. - */ -JNIEXPORT jint JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_exitValue( - JNIEnv *env, jobject objSelf) { - - DWORD exitCode; - HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); - GetExitCodeProcess(hProcess, &exitCode); - LogDebugMessage(L"GetExitCodeProcess: %x :%d\n", hProcess, exitCode); - - return exitCode; -} - - -/* - * native void dispose(); - * - * The "00024" in the function name is an artifact of how JNI encodes - * special characters. U+0024 is '$'. - */ -JNIEXPORT void JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_dispose( - JNIEnv *env, jobject objSelf) { - - HANDLE hProcess, hThread; - - jboolean disposed = (*env)->GetBooleanField(env, objSelf, wps_disposed); - - if (JNI_TRUE != disposed) { - hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); - hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread); - - CloseHandle(hProcess); - CloseHandle(hThread); - (*env)->SetBooleanField(env, objSelf, wps_disposed, JNI_TRUE); - LogDebugMessage(L"disposed: %p\n", objSelf); - } -} - - -/* - * native static FileDescriptor getFileDescriptorFromHandle(long handle); - * - * The "00024" in the function name is an artifact of how JNI encodes - * special characters. U+0024 is '$'. - */ -JNIEXPORT jobject JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_00024WinutilsProcessStub_getFileDescriptorFromHandle( - JNIEnv *env, jclass klass, jlong handle) { - - LogDebugMessage(L"getFileDescriptorFromHandle: %x\n", handle); - return fd_create(env, (long) handle); -} - diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.h deleted file mode 100644 index 6ab8ad6..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/winutils_process_stub.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - - -#define WINUTILS_PROCESS_STUB_CLASS "org/apache/hadoop/io/nativeio/NativeIO$WinutilsProcessStub" - -void winutils_process_stub_init(JNIEnv *env); -void winutils_process_stub_deinit(JNIEnv *env); -jobject winutils_process_stub_create(JNIEnv *env, - jlong hProcess, jlong hThread, jlong hStdIn, jlong hStdOut, jlong hStdErr); - - diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/client.c b/hadoop-common-project/hadoop-common/src/main/winutils/client.c deleted file mode 100644 index 5509d2d..0000000 --- a/hadoop-common-project/hadoop-common/src/main/winutils/client.c +++ /dev/null @@ -1,162 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with this -* work for additional information regarding copyright ownership. The ASF -* licenses this file to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -* License for the specific language governing permissions and limitations under -* the License. -*/ - -#include "winutils.h" -#include -#include -#include "hdpwinutilsvc_h.h" - -#pragma comment(lib, "Rpcrt4.lib") -#pragma comment(lib, "advapi32.lib") - -static ACCESS_MASK CLIENT_MASK = 1; - - -VOID ReportClientError(LPWSTR lpszLocation, DWORD dwError) { - LPWSTR debugMsg = NULL; - int len; - WCHAR hexError[32]; - HRESULT hr; - - if (IsDebuggerPresent()) { - len = FormatMessageW( - FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, - NULL, dwError, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPWSTR)&debugMsg, 0, NULL); - - LogDebugMessage(L"%s: %s: %x: %.*s\n", GetSystemTimeString(), lpszLocation, dwError, len, debugMsg); - } - - if (NULL != debugMsg) LocalFree(debugMsg); -} - -DWORD RpcCall_TaskCreateAsUser( - LPCWSTR cwd, LPCWSTR jobName, - LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine, - HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr) -{ - DWORD dwError = EXIT_FAILURE; - RPC_STATUS status; - LPWSTR lpszStringBinding = NULL; - ULONG ulCode; - DWORD dwSelfPid = GetCurrentProcessId(); - CREATE_PROCESS_REQUEST request; - CREATE_PROCESS_RESPONSE *response = NULL; - RPC_SECURITY_QOS_V3 qos; - PSID pLocalSystemSid = NULL; - SID_IDENTIFIER_AUTHORITY authNT = SECURITY_NT_AUTHORITY; - - if (!AllocateAndInitializeSid(&authNT, 1, - SECURITY_LOCAL_SYSTEM_RID, - 0, 0, 0, 0, 0, 0, 0, - &pLocalSystemSid)) { - dwError = GetLastError(); - ReportClientError(L"AllocateAndInitializeSid", dwError); - goto done; - } - - ZeroMemory(&qos, sizeof(qos)); - qos.Version = RPC_C_SECURITY_QOS_VERSION_3; - qos.Capabilities = RPC_C_QOS_CAPABILITIES_LOCAL_MA_HINT | RPC_C_QOS_CAPABILITIES_MUTUAL_AUTH; - qos.IdentityTracking = RPC_C_QOS_IDENTITY_DYNAMIC; - qos.ImpersonationType = RPC_C_IMP_LEVEL_DEFAULT; - qos.Sid = pLocalSystemSid; - - ZeroMemory(&request, sizeof(request)); - request.cwd = cwd; - request.jobName = jobName; - request.user = user; - request.pidFile = pidFile; - request.cmdLine = cmdLine; - - status = RpcStringBindingCompose(NULL, - SVCBINDING, - NULL, - SVCNAME, - NULL, - &lpszStringBinding); - if (RPC_S_OK != status) { - ReportClientError(L"RpcStringBindingCompose", status); - dwError = status; - goto done; - } - - status = RpcBindingFromStringBinding(lpszStringBinding, &hHadoopWinutilsSvcBinding); - - if (RPC_S_OK != status) { - ReportClientError(L"RpcBindingFromStringBinding", status); - dwError = status; - goto done; - } - - status = RpcBindingSetAuthInfoEx( - hHadoopWinutilsSvcBinding, - NULL, - RPC_C_AUTHN_LEVEL_PKT_PRIVACY, // AuthnLevel - RPC_C_AUTHN_WINNT, // AuthnSvc - NULL, // AuthnIdentity (self) - RPC_C_AUTHZ_NONE, // AuthzSvc - &qos); - if (RPC_S_OK != status) { - ReportClientError(L"RpcBindingSetAuthInfoEx", status); - dwError = status; - goto done; - } - - RpcTryExcept { - dwError = WinutilsCreateProcessAsUser(dwSelfPid, &request, &response); - } - RpcExcept(1) { - ulCode = RpcExceptionCode(); - ReportClientError(L"RpcExcept", ulCode); - dwError = (DWORD) ulCode; - } - RpcEndExcept; - - if (ERROR_SUCCESS == dwError) { - *phProcess = response->hProcess; - *phThread = response->hThread; - *phStdIn = response->hStdIn; - *phStdOut = response->hStdOut; - *phStdErr = response->hStdErr; - } - - // From here on forward we do no change dwError even on RPC cleanup errors - status = RpcBindingFree(&hHadoopWinutilsSvcBinding); - if (RPC_S_OK != status) { - ReportClientError(L"RpcBindingFree", status); - goto done; - } - -done: - if (pLocalSystemSid) FreeSid(pLocalSystemSid); - - if (NULL != response) { - MIDL_user_free(response); - } - - if (NULL != lpszStringBinding) { - status = RpcStringFree(&lpszStringBinding); - if (RPC_S_OK != status) { - ReportClientError(L"RpcStringFree", status); - } - } - - return dwError; -} - diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/config.cpp b/hadoop-common-project/hadoop-common/src/main/winutils/config.cpp deleted file mode 100644 index 8524f7c..0000000 --- a/hadoop-common-project/hadoop-common/src/main/winutils/config.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with this -* work for additional information regarding copyright ownership. The ASF -* licenses this file to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -* License for the specific language governing permissions and limitations under -* the License. -*/ - -#include "winutils.h" -#include -#import "msxml6.dll" - - -#define YARN_SITE_XML_PATH L"%HADOOP_CONF_DIR%\\yarn-site.xml" -#define YARN_DEFAULT_XML_PATH L"%HADOOP_CONF_DIR%\\yarn-default.xml" - - -#define ERROR_CHECK_HRESULT_DONE(hr, message) \ - if (FAILED(hr)) { \ - dwError = (DWORD) hr; \ - LogDebugMessage(L"%s: %x", message, hr); \ - goto done; \ - } - -DWORD GetConfigValue(__in LPCWSTR keyName, - __out size_t* len, __out_bcount(len) LPCWSTR* value) { - - DWORD dwError = ERROR_SUCCESS; - WCHAR xmlPath[MAX_PATH]; - - *len = 0; - *value = NULL; - - if (0 == ExpandEnvironmentStrings(YARN_SITE_XML_PATH, xmlPath, MAX_PATH)) { - dwError = GetLastError(); - goto done; - } - - dwError = GetConfigValueFromXmlFile(xmlPath, keyName, len, value); - if (*len) { - goto done; - } - - if (0 == ExpandEnvironmentStrings(YARN_DEFAULT_XML_PATH, xmlPath, MAX_PATH)) { - dwError = GetLastError(); - goto done; - } - - dwError = GetConfigValueFromXmlFile(xmlPath, keyName, len, value); - -done: - if (*len) { - LogDebugMessage(L"GetConfigValue:%d key:%s len:%d value:%.*s from:%s\n", dwError, keyName, *len, *len, *value, xmlPath); - } - return dwError; -} - - -DWORD GetConfigValueFromXmlFile(__in LPCWSTR xmlFile, __in LPCWSTR keyName, - __out size_t* outLen, __out_bcount(len) LPCWSTR* outValue) { - - DWORD dwError = ERROR_SUCCESS; - HRESULT hr; - WCHAR keyXsl[8192]; - size_t len = 0; - LPWSTR value = NULL; - - *outLen = 0; - *outValue = NULL; - - hr = StringCbPrintf(keyXsl, sizeof(keyXsl), L"//configuration/property[name='%s']/value/text()", keyName); - ERROR_CHECK_HRESULT_DONE(hr, L"StringCbPrintf"); - - hr = CoInitialize(NULL); - ERROR_CHECK_HRESULT_DONE(hr, L"CoInitialize"); - - try { - MSXML2::IXMLDOMDocument2Ptr pDoc; - hr = pDoc.CreateInstance(__uuidof(MSXML2::DOMDocument60), NULL, CLSCTX_INPROC_SERVER); - ERROR_CHECK_HRESULT_DONE(hr, L"CreateInstance"); - - pDoc->async = VARIANT_FALSE; - pDoc->validateOnParse = VARIANT_FALSE; - pDoc->resolveExternals = VARIANT_FALSE; - - _variant_t file(xmlFile); - - if (VARIANT_FALSE == pDoc->load(file)) { - dwError = pDoc->parseError->errorCode; - LogDebugMessage(L"load %s failed:%d %s\n", xmlFile, dwError, - static_cast(pDoc->parseError->Getreason())); - goto done; - } - - MSXML2::IXMLDOMElementPtr pRoot = pDoc->documentElement; - MSXML2::IXMLDOMNodePtr keyNode = pRoot->selectSingleNode(keyXsl); - - if (keyNode) { - _bstr_t bstrValue = static_cast<_bstr_t>(keyNode->nodeValue); - len = bstrValue.length(); - value = (LPWSTR) LocalAlloc(LPTR, (len+1) * sizeof(WCHAR)); - LPCWSTR lpwszValue = static_cast(bstrValue); - memcpy(value, lpwszValue, (len) * sizeof(WCHAR)); - LogDebugMessage(L"key:%s :%.*s [%s]\n", keyName, len, value, lpwszValue); - *outLen = len; - *outValue = value; - } - else { - LogDebugMessage(L"node Xpath:%s not found in:%s\n", keyXsl, xmlFile); - } - } - catch(_com_error errorObject) { - dwError = errorObject.Error(); - LogDebugMessage(L"catch _com_error:%x %s\n", dwError, errorObject.ErrorMessage()); - goto done; - } - -done: - CoUninitialize(); - - return dwError; -} - - diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/hdpwinutilsvc.idl b/hadoop-common-project/hadoop-common/src/main/winutils/hdpwinutilsvc.idl deleted file mode 100644 index 2285178..0000000 --- a/hadoop-common-project/hadoop-common/src/main/winutils/hdpwinutilsvc.idl +++ /dev/null @@ -1,35 +0,0 @@ -import "oaidl.idl"; -import "ocidl.idl"; - -[ - uuid(0492311C-1718-4F53-A6EB-86AD7039988D), - version(1.0), - pointer_default(unique), - implicit_handle(handle_t hHadoopWinutilsSvcBinding), - endpoint("ncalrpc:[hadoopwinutilsvc]"), -] -interface HadoopWinutilSvc -{ - typedef struct { - [string] const wchar_t* cwd; - [string] const wchar_t* jobName; - [string] const wchar_t* user; - [string] const wchar_t* pidFile; - [string] const wchar_t* cmdLine; - } CREATE_PROCESS_REQUEST; - - typedef struct { - LONG_PTR hProcess; - LONG_PTR hThread; - LONG_PTR hStdIn; - LONG_PTR hStdOut; - LONG_PTR hStdErr; - } CREATE_PROCESS_RESPONSE; - - - error_status_t WinutilsCreateProcessAsUser( - [in] int nmPid, - [in] CREATE_PROCESS_REQUEST *request, - [out] CREATE_PROCESS_RESPONSE **response); - -} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h b/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h index 7f528b1..cc4d038 100644 --- a/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h +++ b/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h @@ -182,8 +182,6 @@ DWORD LoadUserProfileForLogon(__in HANDLE logonHandle, __out PROFILEINFO * pi); DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi); -DWORD RunService(__in int argc, __in_ecount(argc) wchar_t *argv[]); -void ServiceUsage(); LPCWSTR GetSystemTimeString(); @@ -193,15 +191,12 @@ DWORD SplitStringIgnoreSpaceW(__in size_t len, __in_bcount(len) LPCWSTR source, __in WCHAR deli, __out size_t* count, __out_ecount(count) WCHAR*** out); -DWORD GetConfigValue( - __in LPCWSTR keyName, - __out size_t* len, - __out_bcount(len) LPCWSTR* value); -DWORD GetConfigValueFromXmlFile( - __in LPCWSTR xmlFile, - __in LPCWSTR keyName, - __out size_t* len, - __out_bcount(len) LPCWSTR* value); + +DWORD GetAbsolutePathFromModule( + __in LPCWSTR relPath, + __in size_t size, + __out_bcount(size) LPWSTR buffer); + DWORD BuildServiceSecurityDescriptor( diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c index 1c6237d..f80f59d 100644 --- a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c +++ b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c @@ -2409,6 +2409,58 @@ done: } +DWORD GetAbsolutePathFromModule( + __in LPCWSTR relPath, + __in size_t size, + __out_bcount(size) LPWSTR buffer) +{ + WCHAR selfModuleName[MAX_PATH]; + WCHAR selfModulePath[MAX_PATH]; + WCHAR relativePath[MAX_PATH]; + DWORD dwError = ERROR_SUCCESS; + HRESULT hr = S_OK; + + // Get the exe startup path (not the current CWD) + // + GetModuleFileName(NULL, selfModuleName, MAX_PATH); + dwError = GetLastError(); // Always check after GetModuleFileName for ERROR_INSSUFICIENT_BUFFER + if (dwError) { + goto done; + } + + // drop the filename, keep the path + // + GetFullPathName(selfModuleName, MAX_PATH, selfModulePath, NULL); + dwError = GetLastError(); + if (dwError) { + goto done; + } + + // Compose with the relative path provided + // + hr = StringCbPrintf(relativePath, MAX_PATH, L"%s\\%s", selfModulePath, relPath); + if (FAILED(hr)) { + // There is no reliable HRESULT to WIN32 mapping, use code. + // see http://blogs.msdn.com/b/oldnewthing/archive/2006/11/03/942851.aspx + // + dwError = HRESULT_CODE(hr); + goto done; + } + + // Make it an absolute path + // + if (NULL == _wfullpath(buffer, relativePath, size)) { + LogDebugMessage(L"_wfullpath errno: %d\n", errno); + dwError = ERROR_BAD_PATHNAME; + goto done; + } + +done: + return dwError; +} + + + //---------------------------------------------------------------------------- // Function: MIDL_user_allocate // diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj index 63d936b..fc0519d 100644 --- a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj +++ b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj @@ -160,22 +160,11 @@ - - - - - - - - true - X64 - - diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/main.c b/hadoop-common-project/hadoop-common/src/main/winutils/main.c index e43b6ac..3c2324f 100644 --- a/hadoop-common-project/hadoop-common/src/main/winutils/main.c +++ b/hadoop-common-project/hadoop-common/src/main/winutils/main.c @@ -67,10 +67,6 @@ int wmain(__in int argc, __in_ecount(argc) wchar_t* argv[]) { return SystemInfo(); } - else if (wcscmp(L"service", cmd) == 0) - { - return RunService(argc - 1, argv + 1); - } else if (wcscmp(L"help", cmd) == 0) { Usage(argv[0]); @@ -124,8 +120,5 @@ The available commands and their usages are:\n\n", program); fwprintf(stdout, L"%-15s%s\n\n", L"task", L"Task operations."); TaskUsage(); - fwprintf(stdout, L"%-15s%s\n\n", L"service", L"Service operations."); - ServiceUsage(); - fwprintf(stdout, L"\n\n"); } diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/service.c b/hadoop-common-project/hadoop-common/src/main/winutils/service.c deleted file mode 100644 index d8e6ac6..0000000 --- a/hadoop-common-project/hadoop-common/src/main/winutils/service.c +++ /dev/null @@ -1,883 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with this -* work for additional information regarding copyright ownership. The ASF -* licenses this file to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -* License for the specific language governing permissions and limitations under -* the License. -*/ - -#include "winutils.h" -#include "winutils_msg.h" -#include -#include -#include -#include -#include -#include -#include "hdpwinutilsvc_h.h" - -#pragma comment(lib, "Rpcrt4.lib") -#pragma comment(lib, "advapi32.lib") -#pragma comment(lib, "authz.lib") - - -#define NM_WSCE_ALLOWED L"yarn.nodemanager.windows-secure-container-executor.allowed" -#define NM_WSCE_DENIED L"yarn.nodemanager.windows-secure-container-executor.denied" - -#define SERVICE_ACCESS_MASK 0x00000001 - -SERVICE_STATUS gSvcStatus; -SERVICE_STATUS_HANDLE gSvcStatusHandle; -HANDLE ghSvcStopEvent = INVALID_HANDLE_VALUE; -HANDLE ghWaitObject = INVALID_HANDLE_VALUE; -HANDLE ghEventLog = INVALID_HANDLE_VALUE; -BOOL isListenning = FALSE; -PSECURITY_DESCRIPTOR pAllowedSD = NULL; - -VOID SvcError(DWORD dwError); -VOID WINAPI SvcMain(DWORD dwArg, LPTSTR* lpszArgv); -DWORD SvcInit(); -DWORD RpcInit(); -DWORD AuthInit(); -VOID ReportSvcStatus( DWORD dwCurrentState, - DWORD dwWin32ExitCode, - DWORD dwWaitHint); -VOID WINAPI SvcCtrlHandler( DWORD dwCtrl ); -VOID CALLBACK SvcShutdown( - _In_ PVOID lpParameter, - _In_ BOOLEAN TimerOrWaitFired); - -#define CHECK_ERROR_DONE(status, expected, category, message) \ - if (status != expected) { \ - ReportSvcCheckError( \ - EVENTLOG_ERROR_TYPE, \ - category, \ - status, \ - message); \ - goto done; \ - } else { \ - LogDebugMessage(L"%s: OK\n", message); \ - } - - -#define CHECK_RPC_STATUS_DONE(status, message) \ - CHECK_ERROR_DONE(status, RPC_S_OK, SERVICE_CATEGORY, message) - -#define CHECK_SVC_STATUS_DONE(status, message) \ - CHECK_ERROR_DONE(status, ERROR_SUCCESS, SERVICE_CATEGORY, message) - -#define CHECK_UNWIND_RPC(rpcCall) { \ - unwindStatus = rpcCall; \ - if (RPC_S_OK != unwindStatus) { \ - ReportSvcCheckError( \ - EVENTLOG_WARNING_TYPE, \ - SERVICE_CATEGORY, \ - unwindStatus, \ - L#rpcCall); \ - } \ - } - - -//---------------------------------------------------------------------------- -// Function: ReportSvcCheckError -// -// Description: -// Reports an error with the system event log and to debugger console (if present) -// -void ReportSvcCheckError(WORD type, WORD category, DWORD dwError, LPCWSTR message) { - int len; - LPWSTR systemMsg = NULL; - LPWSTR appMsg = NULL; - DWORD dwReportError; - LPWSTR reportMsg = NULL; - WCHAR hexError[32]; - LPCWSTR inserts[] = {message, NULL, NULL, NULL}; - HRESULT hr; - - hr = StringCbPrintf(hexError, sizeof(hexError), TEXT("%x"), dwError); - if (SUCCEEDED(hr)) { - inserts[1] = hexError; - } - else { - inserts[1] = L"(Failed to format dwError as string)"; - } - - len = FormatMessageW( - FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, - NULL, dwError, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPWSTR)&systemMsg, 0, NULL); - - if (len) { - inserts[2] = systemMsg; - } - else { - inserts[2] = L"(Failed to get the system error message)"; - } - - LogDebugMessage(L"%s:%d %.*s\n", message, dwError, len, systemMsg); - - if (INVALID_HANDLE_VALUE != ghEventLog) { - if (!ReportEvent(ghEventLog, type, category, MSG_CHECK_ERROR, - NULL, // lpUserSid - (WORD) 3, // wNumStrings - (DWORD) 0, // dwDataSize - inserts, // *lpStrings - NULL // lpRawData - )) { - // We tried to report and failed. Send to dbg. - dwReportError = GetLastError(); - len = FormatMessageW( - FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, - NULL, dwReportError, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPWSTR)&reportMsg, 0, NULL); - LogDebugMessage(L"ReportEvent: Error:%d %.*s\n", dwReportError, reportMsg); - } - }; - - if (NULL != systemMsg) LocalFree(systemMsg); - if (NULL != reportMsg) LocalFree(reportMsg); -} - - -VOID ReportSvcMessage(WORD type, WORD category, DWORD msgId) { - DWORD dwError; - - if (INVALID_HANDLE_VALUE != ghEventLog) { - if (!ReportEvent(ghEventLog, type, category, msgId, - NULL, // lpUserSid - (WORD) 0, // wNumStrings - (DWORD) 0, // dwDataSize - NULL, // *lpStrings - NULL // lpRawData - )) { - // We tried to report and failed but debugger is attached. Send to dbg. - dwError = GetLastError(); - LogDebugMessage(L"ReportEvent: error %d\n", dwError); - } - } -} - - -//---------------------------------------------------------------------------- -// Function: RunService -// -// Description: -// Registers with NT SCM and starts the service -// -// Returns: -// ERROR_SUCCESS: On success -// Error code otherwise: otherwise -DWORD RunService(__in int argc, __in_ecount(argc) wchar_t *argv[]) -{ - DWORD dwError= ERROR_SUCCESS; - int argStart = 1; - - static const SERVICE_TABLE_ENTRY serviceTable[] = { - { SVCNAME, (LPSERVICE_MAIN_FUNCTION) SvcMain }, - { NULL, NULL } - }; - - dwError = AuthInit(); - if (ERROR_SUCCESS != dwError) { - SvcError(dwError); - goto done; - } - - ghEventLog = RegisterEventSource(NULL, SVCNAME); - if (NULL == ghEventLog) { - dwError = GetLastError(); - CHECK_SVC_STATUS_DONE(dwError, L"RegisterEventSource") - } - - if (!StartServiceCtrlDispatcher(serviceTable)) { - dwError = GetLastError(); - CHECK_SVC_STATUS_DONE(dwError, L"StartServiceCtrlDispatcher") - } - -done: - return dwError; -} - -//---------------------------------------------------------------------------- -// Function: SvcMain -// -// Description: -// Service main entry point. -// -VOID WINAPI SvcMain() { - DWORD dwError = ERROR_SUCCESS; - - gSvcStatusHandle = RegisterServiceCtrlHandler( - SVCNAME, - SvcCtrlHandler); - if( !gSvcStatusHandle ) { - dwError = GetLastError(); - CHECK_SVC_STATUS_DONE(dwError, L"RegisterServiceCtrlHandler") - } - - // These SERVICE_STATUS members remain as set here - gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; - gSvcStatus.dwServiceSpecificExitCode = 0; - - // Report initial status to the SCM - ReportSvcStatus( SERVICE_START_PENDING, NO_ERROR, 3000 ); - - // Perform service-specific initialization and work. - dwError = SvcInit(); - -done: - return; -} - -//---------------------------------------------------------------------------- -// Function: SvcInit -// -// Description: -// Initializes the service. -// -DWORD SvcInit() { - DWORD dwError = ERROR_SUCCESS; - - dwError = EnablePrivilege(SE_DEBUG_NAME); - if( dwError != ERROR_SUCCESS ) { - goto done; - } - - // The recommended way to shutdown the service is to use an event - // and attach a callback with RegisterWaitForSingleObject - // - ghSvcStopEvent = CreateEvent( - NULL, // default security attributes - TRUE, // manual reset event - FALSE, // not signaled - NULL); // no name - - if ( ghSvcStopEvent == NULL) - { - dwError = GetLastError(); - ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, - dwError, L"CreateEvent"); - ReportSvcStatus( SERVICE_STOPPED, dwError, 0 ); - goto done; - } - - if (!RegisterWaitForSingleObject (&ghWaitObject, - ghSvcStopEvent, - SvcShutdown, - NULL, - INFINITE, - WT_EXECUTEONLYONCE)) { - dwError = GetLastError(); - ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, - dwError, L"RegisterWaitForSingleObject"); - CloseHandle(ghSvcStopEvent); - ReportSvcStatus( SERVICE_STOPPED, dwError, 0 ); - goto done; - } - - // Report running status when initialization is complete. - ReportSvcStatus( SERVICE_RUNNING, NO_ERROR, 0 ); - - dwError = RpcInit(); - -done: - return dwError; -} - -//---------------------------------------------------------------------------- -// Function: RpcAuthorizeCallback -// -// Description: -// RPC Authorization callback. -// -// Returns: -// RPC_S_OK for access authorized -// RPC_S_ACCESS_DENIED for access denied -// -RPC_STATUS CALLBACK RpcAuthorizeCallback ( - RPC_IF_HANDLE hInterface, - void* pContext) -{ - RPC_STATUS status, - unwindStatus, - authStatus = RPC_S_ACCESS_DENIED; - DWORD dwError; - LUID luidReserved2; - AUTHZ_ACCESS_REQUEST request; - AUTHZ_ACCESS_REPLY reply; - AUTHZ_CLIENT_CONTEXT_HANDLE hClientContext = NULL; - DWORD authError = ERROR_SUCCESS; - DWORD saclResult = 0; - ACCESS_MASK grantedMask = 0; - - ZeroMemory(&luidReserved2, sizeof(luidReserved2)); - ZeroMemory(&request, sizeof(request)); - ZeroMemory(&reply, sizeof(reply)); - - status = RpcGetAuthorizationContextForClient(NULL, - FALSE, // ImpersonateOnReturn - NULL, // Reserved1 - NULL, // pExpirationTime - luidReserved2, // Reserved2 - 0, // Reserved3 - NULL, // Reserved4 - &hClientContext); - CHECK_RPC_STATUS_DONE(status, L"RpcGetAuthorizationContextForClient"); - - request.DesiredAccess = MAXIMUM_ALLOWED; - reply.Error = &authError; - reply.SaclEvaluationResults = &saclResult; - reply.ResultListLength = 1; - reply.GrantedAccessMask = &grantedMask; - - if (!AuthzAccessCheck( - 0, - hClientContext, - &request, - NULL, // AuditEvent - pAllowedSD, - NULL, // OptionalSecurityDescriptorArray - 0, // OptionalSecurityDescriptorCount - &reply, - NULL // phAccessCheckResults - )) { - dwError = GetLastError(); - CHECK_SVC_STATUS_DONE(dwError, L"AuthzAccessCheck"); - } - - LogDebugMessage(L"AutzAccessCheck: Error:%d sacl:%d access:%d\n", - authError, saclResult, grantedMask); - if (authError == ERROR_SUCCESS && (grantedMask & SERVICE_ACCESS_MASK)) { - authStatus = RPC_S_OK; - } - -done: - if (NULL != hClientContext) CHECK_UNWIND_RPC(RpcFreeAuthorizationContext(&hClientContext)); - return authStatus; -} - -//---------------------------------------------------------------------------- -// Function: AuthInit -// -// Description: -// Initializes the authorization structures (security descriptor). -// -// Notes: -// This is called from RunService solely for debugging purposed -// so that it can be tested by wimply running winutil service from CLI (no SCM) -// -DWORD AuthInit() { - DWORD dwError = ERROR_SUCCESS; - int count = 0; - int crt = 0; - int len = 0; - LPCWSTR value = NULL; - WCHAR** tokens = NULL; - LPWSTR lpszSD = NULL; - ULONG cchSD = 0; - DWORD dwBufferSize = 0; - int allowedCount = 0; - PSID* allowedSids = NULL; - - - dwError = GetConfigValue(NM_WSCE_ALLOWED, &len, &value); - CHECK_SVC_STATUS_DONE(dwError, L"GetConfigValue"); - - if (0 == len) { - CHECK_SVC_STATUS_DONE(ERROR_BAD_CONFIGURATION, NM_WSCE_ALLOWED); - } - - dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens); - CHECK_SVC_STATUS_DONE(dwError, L"SplitStringIgnoreSpaceW"); - - allowedSids = (PSID*) LocalAlloc(LPTR, sizeof(PSID) * count); - for (crt = 0; crt < count; ++crt) { - dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]); - CHECK_SVC_STATUS_DONE(dwError, L"GetSidFromAcctNameW"); - } - - allowedCount = count; - - dwError = BuildServiceSecurityDescriptor(SERVICE_ACCESS_MASK, - allowedCount, allowedSids, 0, NULL, &pAllowedSD); - CHECK_SVC_STATUS_DONE(dwError, L"BuildServiceSecurityDescriptor"); - -done: - if (lpszSD) LocalFree(lpszSD); - if (value) LocalFree(value); - if (tokens) LocalFree(tokens); - return dwError; -} - -//---------------------------------------------------------------------------- -// Function: RpcInit -// -// Description: -// Initializes the RPC infrastructure and starts the RPC listenner. -// -DWORD RpcInit() { - RPC_STATUS status; - DWORD dwError; - - status = RpcServerUseProtseqIf(SVCBINDING, - RPC_C_LISTEN_MAX_CALLS_DEFAULT, - HadoopWinutilSvc_v1_0_s_ifspec, - NULL); - if (RPC_S_OK != status) { - ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, - status, L"RpcServerUseProtseqIf"); - SvcError(status); - dwError = status; - goto done; - } - - status = RpcServerRegisterIfEx(HadoopWinutilSvc_v1_0_s_ifspec, - NULL, // MgrTypeUuid - NULL, // MgrEpv - RPC_IF_ALLOW_LOCAL_ONLY, // Flags - RPC_C_LISTEN_MAX_CALLS_DEFAULT, // Max calls - RpcAuthorizeCallback); // Auth callback - - if (RPC_S_OK != status) { - ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, - status, L"RpcServerRegisterIfEx"); - SvcError(status); - dwError = status; - goto done; - } - - status = RpcServerListen(1, RPC_C_LISTEN_MAX_CALLS_DEFAULT, TRUE); - if (RPC_S_ALREADY_LISTENING == status) { - ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY, - status, L"RpcServerListen"); - } - else if (RPC_S_OK != status) { - ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, - status, L"RpcServerListen"); - SvcError(status); - dwError = status; - goto done; - } - - isListenning = TRUE; - - ReportSvcMessage(EVENTLOG_INFORMATION_TYPE, SERVICE_CATEGORY, - MSG_RPC_SERVICE_HAS_STARTED); - -done: - return dwError; -} - -//---------------------------------------------------------------------------- -// Function: RpcStop -// -// Description: -// Tears down the RPC infrastructure and stops the RPC listenner. -// -VOID RpcStop() { - RPC_STATUS status; - - if (isListenning) { - - status = RpcMgmtStopServerListening(NULL); - isListenning = FALSE; - - if (RPC_S_OK != status) { - ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY, - status, L"RpcMgmtStopServerListening"); - } - - ReportSvcMessage(EVENTLOG_INFORMATION_TYPE, SERVICE_CATEGORY, - MSG_RPC_SERVICE_HAS_STOPPED); - } -} - -//---------------------------------------------------------------------------- -// Function: CleanupHandles -// -// Description: -// Cleans up the global service handles. -// -VOID CleanupHandles() { - if (INVALID_HANDLE_VALUE != ghWaitObject) { - UnregisterWait(ghWaitObject); - ghWaitObject = INVALID_HANDLE_VALUE; - } - if (INVALID_HANDLE_VALUE != ghSvcStopEvent) { - CloseHandle(ghSvcStopEvent); - ghSvcStopEvent = INVALID_HANDLE_VALUE; - } - if (INVALID_HANDLE_VALUE != ghEventLog) { - DeregisterEventSource(ghEventLog); - ghEventLog = INVALID_HANDLE_VALUE; - } -} - -//---------------------------------------------------------------------------- -// Function: SvcError -// -// Description: -// Aborts the startup sequence. Reports error, stops RPC, cleans up globals. -// -VOID SvcError(DWORD dwError) { - RpcStop(); - CleanupHandles(); - ReportSvcStatus( SERVICE_STOPPED, dwError, 0 ); -} - -//---------------------------------------------------------------------------- -// Function: SvcShutdown -// -// Description: -// Callback when the shutdown event is signaled. Stops RPC, cleans up globals. -// -VOID CALLBACK SvcShutdown( - _In_ PVOID lpParameter, - _In_ BOOLEAN TimerOrWaitFired) { - RpcStop(); - CleanupHandles(); - ReportSvcStatus( SERVICE_STOPPED, NO_ERROR, 0 ); -} - -//---------------------------------------------------------------------------- -// Function: SvcCtrlHandler -// -// Description: -// Callback from SCM for for service events (signals). -// -// Notes: -// Shutdown is indirect, we set her the STOP_PENDING state and signal the stop event. -// Signaling the event invokes SvcShutdown which completes the shutdown. -// This two staged approach allows the SCM handler to complete fast, -// not blocking the SCM big fat global lock. -// -VOID WINAPI SvcCtrlHandler( DWORD dwCtrl ) -{ - // Handle the requested control code. - - switch(dwCtrl) - { - case SERVICE_CONTROL_STOP: - ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0); - - // Signal the service to stop. - SetEvent(ghSvcStopEvent); - - return; - - default: - break; - } - -} - -//---------------------------------------------------------------------------- -// Function: ReportSvcStatus -// -// Description: -// Updates the service status with the SCM. -// -VOID ReportSvcStatus( DWORD dwCurrentState, - DWORD dwWin32ExitCode, - DWORD dwWaitHint) -{ - static DWORD dwCheckPoint = 1; - DWORD dwError; - - // Fill in the SERVICE_STATUS structure. - - gSvcStatus.dwCurrentState = dwCurrentState; - gSvcStatus.dwWin32ExitCode = dwWin32ExitCode; - gSvcStatus.dwWaitHint = dwWaitHint; - - if (dwCurrentState == SERVICE_START_PENDING) - gSvcStatus.dwControlsAccepted = 0; - else gSvcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP; - - if ( (dwCurrentState == SERVICE_RUNNING) || - (dwCurrentState == SERVICE_STOPPED) ) - gSvcStatus.dwCheckPoint = 0; - else gSvcStatus.dwCheckPoint = dwCheckPoint++; - - // Report the status of the service to the SCM. - if (!SetServiceStatus( gSvcStatusHandle, &gSvcStatus)) { - dwError = GetLastError(); - ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY, - dwError, L"SetServiceStatus"); - }; -} - -//---------------------------------------------------------------------------- -// Function: WinutilsCreateProcessAsUser -// -// Description: -// The RPC midl declared function implementation -// -// Returns: -// ERROR_SUCCESS: On success -// Error code otherwise: otherwise -// -// Notes: -// This is the entry point when the NodeManager does the RPC call -// Note that the RPC call does not do any S4U work. Is simply spawns (suspended) wintutils -// using the right command line and the handles over the spwaned process to the NM -// The actual S4U work occurs in the spawned process, run and monitored by the NM -// -error_status_t WinutilsCreateProcessAsUser( - /* [in] */ int nmPid, - /* [in] */ CREATE_PROCESS_REQUEST *request, - /* [out] */ CREATE_PROCESS_RESPONSE **response) { - - DWORD dwError = ERROR_SUCCESS; - LPCWSTR inserts[] = {request->cwd, request->jobName, request->user, request->pidFile, request->cmdLine, NULL}; - WCHAR winutilsPath[MAX_PATH]; - WCHAR fullCmdLine[32768]; - HANDLE taskStdInRd = INVALID_HANDLE_VALUE, taskStdInWr = INVALID_HANDLE_VALUE, - taskStdOutRd = INVALID_HANDLE_VALUE, taskStdOutWr = INVALID_HANDLE_VALUE, - taskStdErrRd = INVALID_HANDLE_VALUE, taskStdErrWr = INVALID_HANDLE_VALUE, - hNmProcess = INVALID_HANDLE_VALUE, - hDuplicateProcess = INVALID_HANDLE_VALUE, - hDuplicateThread = INVALID_HANDLE_VALUE, - hDuplicateStdIn = INVALID_HANDLE_VALUE, - hDuplicateStdOut = INVALID_HANDLE_VALUE, - hDuplicateStdErr = INVALID_HANDLE_VALUE, - hSelfProcess = INVALID_HANDLE_VALUE; - BOOL fMustCleanupProcess = FALSE; - - HRESULT hr; - STARTUPINFO si; - PROCESS_INFORMATION pi; - SECURITY_ATTRIBUTES saTaskStdInOutErr; - - ZeroMemory( &si, sizeof(si) ); - si.cb = sizeof(si); - ZeroMemory( &pi, sizeof(pi) ); - pi.hProcess = INVALID_HANDLE_VALUE; - pi.hThread = INVALID_HANDLE_VALUE; - ZeroMemory( &saTaskStdInOutErr, sizeof(saTaskStdInOutErr)); - - // NB: GetCurrentProcess returns a pseudo-handle that just so happens - // has the value -1, ie. INVALID_HANDLE_VALUE. It cannot fail. - // - hSelfProcess = GetCurrentProcess(); - - hNmProcess = OpenProcess(PROCESS_DUP_HANDLE, FALSE, nmPid); - if (NULL == hNmProcess) { - dwError = GetLastError(); - goto done; - } - - GetModuleFileName(NULL, winutilsPath, sizeof(winutilsPath)/sizeof(WCHAR)); - dwError = GetLastError(); // Always check after GetModuleFileName for ERROR_INSSUFICIENT_BUFFER - if (dwError) { - ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, - dwError, L"GetModuleFileName"); - goto done; - } - - // NB. We can call CreateProcess("wintuls","task create ...") or we can call - // CreateProcess(NULL, "winutils task create"). Only the second form passes "task" as - // argv[1], as expected by main. First form passes "task" as argv[0] and main fails. - - hr = StringCbPrintf(fullCmdLine, sizeof(fullCmdLine), L"\"%s\" task createAsUser %ls %ls %ls %ls", - winutilsPath, - request->jobName, request->user, request->pidFile, request->cmdLine); - if (FAILED(hr)) { - ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, - hr, L"StringCbPrintf:fullCmdLine"); - goto done; - } - - LogDebugMessage(L"[%ls]: %ls %ls\n", request->cwd, winutilsPath, fullCmdLine); - - // stdin/stdout/stderr redirection is handled here - // We create 3 anonimous named pipes. - // Security attributes are required so that the handles can be inherited. - // We assign one end of the pipe to the process (stdin gets a read end, stdout gets a write end) - // We then duplicate the other end in the NM process, and we close our own handle - // Finally we return the duplicate handle values to the NM - // The NM will attach Java file dscriptors to the duplicated handles and - // read/write them as ordinary Java InputStream/OutputStream objects - - si.dwFlags |= STARTF_USESTDHANDLES; - - saTaskStdInOutErr.nLength = sizeof(SECURITY_ATTRIBUTES); - saTaskStdInOutErr.bInheritHandle = TRUE; - saTaskStdInOutErr.lpSecurityDescriptor = NULL; - - if (!CreatePipe(&taskStdInRd, &taskStdInWr, &saTaskStdInOutErr, 0)) { - dwError = GetLastError(); - goto done; - } - if (!SetHandleInformation(taskStdInWr, HANDLE_FLAG_INHERIT, FALSE)) { - dwError = GetLastError(); - goto done; - } - si.hStdInput = taskStdInRd; - - if (!CreatePipe(&taskStdOutRd, &taskStdOutWr, &saTaskStdInOutErr, 0)) { - dwError = GetLastError(); - goto done; - } - if (!SetHandleInformation(taskStdOutRd, HANDLE_FLAG_INHERIT, FALSE)) { - dwError = GetLastError(); - goto done; - } - si.hStdOutput = taskStdOutWr; - - if (!CreatePipe(&taskStdErrRd, &taskStdErrWr, &saTaskStdInOutErr, 0)) { - dwError = GetLastError(); - goto done; - } - if (!SetHandleInformation(taskStdErrRd, HANDLE_FLAG_INHERIT, FALSE)) { - dwError = GetLastError(); - goto done; - } - si.hStdError = taskStdErrWr; - - if (!CreateProcess( - NULL, // lpApplicationName, - fullCmdLine, // lpCommandLine, - NULL, // lpProcessAttributes, - NULL, // lpThreadAttributes, - TRUE, // bInheritHandles, - CREATE_SUSPENDED, // dwCreationFlags, - NULL, // lpEnvironment, - request->cwd, // lpCurrentDirectory, - &si, // lpStartupInfo - &pi)) { // lpProcessInformation - - dwError = GetLastError(); - ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, - dwError, L"CreateProcess"); - goto done; - } - - fMustCleanupProcess = TRUE; - - LogDebugMessage(L"CreateProcess: pid:%x\n", pi.dwProcessId); - - if (!DuplicateHandle(hSelfProcess, pi.hProcess, hNmProcess, - &hDuplicateProcess, 0, FALSE, DUPLICATE_SAME_ACCESS)) { - dwError = GetLastError(); - LogDebugMessage(L"failed: pi.hProcess\n"); - goto done; - } - - if (!DuplicateHandle(hSelfProcess, pi.hThread, hNmProcess, - &hDuplicateThread, 0, FALSE, DUPLICATE_SAME_ACCESS)) { - dwError = GetLastError(); - LogDebugMessage(L"failed: pi.hThread\n"); - goto done; - } - - if (!DuplicateHandle(hSelfProcess, taskStdInWr, hNmProcess, - &hDuplicateStdIn, 0, FALSE, DUPLICATE_SAME_ACCESS)) { - dwError = GetLastError(); - LogDebugMessage(L"failed: taskStdInWr\n"); - goto done; - } - - if (!DuplicateHandle(hSelfProcess, taskStdOutRd, hNmProcess, - &hDuplicateStdOut, 0, FALSE, DUPLICATE_SAME_ACCESS)) { - dwError = GetLastError(); - LogDebugMessage(L"failed: taskStdOutRd\n"); - goto done; - } - - if (!DuplicateHandle(hSelfProcess, taskStdErrRd, hNmProcess, - &hDuplicateStdErr, 0, FALSE, DUPLICATE_SAME_ACCESS)) { - dwError = GetLastError(); - LogDebugMessage(L"failed: taskStdErrRd\n"); - goto done; - } - - *response = (CREATE_PROCESS_RESPONSE*) MIDL_user_allocate(sizeof(CREATE_PROCESS_RESPONSE)); - if (NULL == *response) { - dwError = ERROR_OUTOFMEMORY; - LogDebugMessage(L"Failed to allocate CREATE_PROCESS_RESPONSE* response\n"); - goto done; - } - - // We're now transfering ownership of the duplicated handles to the caller - // If the RPC call fails *after* this point the handles are leaked inside the NM process - - (*response)->hProcess = hDuplicateProcess; - (*response)->hThread = hDuplicateThread; - (*response)->hStdIn = hDuplicateStdIn; - (*response)->hStdOut = hDuplicateStdOut; - (*response)->hStdErr = hDuplicateStdErr; - - fMustCleanupProcess = FALSE; - -done: - - if (fMustCleanupProcess) { - LogDebugMessage(L"Cleaning process: %d due to error:%d\n", pi.dwProcessId, dwError); - TerminateProcess(pi.hProcess, EXIT_FAILURE); - - // cleanup the duplicate handles inside the NM. - - if (INVALID_HANDLE_VALUE != hDuplicateProcess) { - DuplicateHandle(hNmProcess, hDuplicateProcess, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); - } - if (INVALID_HANDLE_VALUE != hDuplicateThread) { - DuplicateHandle(hNmProcess, hDuplicateThread, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); - } - if (INVALID_HANDLE_VALUE != hDuplicateStdIn) { - DuplicateHandle(hNmProcess, hDuplicateStdIn, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); - } - if (INVALID_HANDLE_VALUE != hDuplicateStdOut) { - DuplicateHandle(hNmProcess, hDuplicateStdOut, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); - } - if (INVALID_HANDLE_VALUE != hDuplicateStdErr) { - DuplicateHandle(hNmProcess, hDuplicateStdErr, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); - } - } - - if (INVALID_HANDLE_VALUE != hSelfProcess) CloseHandle(hSelfProcess); - if (INVALID_HANDLE_VALUE != hNmProcess) CloseHandle(hNmProcess); - if (INVALID_HANDLE_VALUE != taskStdInRd) CloseHandle(taskStdInRd); - if (INVALID_HANDLE_VALUE != taskStdInWr) CloseHandle(taskStdInWr); - if (INVALID_HANDLE_VALUE != taskStdOutRd) CloseHandle(taskStdOutRd); - if (INVALID_HANDLE_VALUE != taskStdOutWr) CloseHandle(taskStdOutWr); - if (INVALID_HANDLE_VALUE != taskStdErrRd) CloseHandle(taskStdErrRd); - if (INVALID_HANDLE_VALUE != taskStdErrWr) CloseHandle(taskStdErrWr); - - - // This is closing our own process/thread handles. - // If the transfer was succesfull the NM has its own duplicates (if any) - if (INVALID_HANDLE_VALUE != pi.hThread) CloseHandle(pi.hThread); - if (INVALID_HANDLE_VALUE != pi.hProcess) CloseHandle(pi.hProcess); - - return dwError; -} - -//---------------------------------------------------------------------------- -// Function: ServiceUsage -// -// Description: -// Prints the CLI arguments for service command. -// -void ServiceUsage() -{ - fwprintf(stdout, L"\ - Usage: service\n\ - Starts the impersonation helper service.\n\ - This should be called from the SCM.\n\ - The impersonation helper service must run as a high privileged account (LocalSystem)\n\ - and is used by the NodeManager to spawn secure containers.\n"); -} - - diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc deleted file mode 100644 index a2e30ad..0000000 --- a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc +++ /dev/null @@ -1,46 +0,0 @@ -; // winutils.mc - -; // EventLog messages for Hadoop winutils service. - - -LanguageNames=(English=0x409:MSG00409) - - -; // The following are the categories of events. - -MessageIdTypedef=WORD - -MessageId=0x1 -SymbolicName=SERVICE_CATEGORY -Language=English -Service Events -. - -MessageId=0x2 -SymbolicName=LOG_CATEGORY -Language=English -Task Events -. - -; // The following are the message definitions. - -MessageIdTypedef=DWORD - -MessageId=0x80 -SymbolicName=MSG_CHECK_ERROR -Language=English -%1. Error %2: %3. -. - -MessageId=0x100 -SymbolicName=MSG_RPC_SERVICE_HAS_STARTED -Language=English -The LPC server is listenning. -. - -MessageId=0x200 -SymbolicName=MSG_RPC_SERVICE_HAS_STOPPED -Language=English -The LPC server has stopped listenning. -. - diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj index f216b71..c9bdd10 100644 --- a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj +++ b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj @@ -117,10 +117,7 @@ Console true - - X64 - ..\..\..\target\winutils\$(Configuration)\ - + @@ -156,40 +153,7 @@ true - - - $(IntermediateOutputPath) - - - Compiling Messages - mc.exe $(TargetName).mc -z $(TargetName)_msg -r $(IntermediateOutputPath) -h $(IntermediateOutputPath) -U - $(IntermediateOutputPath)\$(TargetName)_msg.rc,$(IntermediateOutputPath)\$(TargetName)_msg.h - - - true - X64 - $(IntermediateOutputPath) - true - true - true - 2 - - - - Midl - ClCompile,ResourceCompile - - - - - - - - - - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 370cc36..a238d99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -32,6 +32,9 @@ ${project.parent.parent.basedir} ../etc/hadoop + ${container-executor.conf.dir} + wsce-site.xml + winutils.exe @@ -253,6 +256,33 @@ + org.codehaus.mojo + exec-maven-plugin + + + compile-ms-wsce + compile + + exec + + + msbuild + + ${basedir}/src/main/native/wsce/wsce.sln + /nologo + /p:winutilsdir=${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/winutils + /p:Configuration=Release + /p:WsceConfDir=${wsce.conf.dir} + /p:WsceConfFile=${wsce.conf.file} + /p:WsceWinutilsBin=${wsce.winutils.bin} + /p:BaseIntermediateOutputPath=${project.build.directory}/ + /p:OutDir=${project.build.directory}/bin/ + + + + + + maven-surefire-plugin diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index 572d703..2e5ad75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -19,14 +19,15 @@ import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.io.PrintStream; -import java.io.Reader; import java.net.InetSocketAddress; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -39,11 +40,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.nativeio.NativeIO; -import org.apache.hadoop.io.nativeio.NativeIO.WinutilsProcessStub; -import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.io.nativeio.NativeIOException; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ICommandExecutor; -import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; @@ -55,22 +54,99 @@ private static final Log LOG = LogFactory .getLog(WindowsSecureContainerExecutor.class); + + private static boolean nativeCodeLoaded = false; + + static { + if (Shell.WINDOWS) { + if(LOG.isDebugEnabled()) { + LOG.debug("Trying to load the WSCE native library wsce-client.dll..."); + } + try { + System.loadLibrary("wsce-client"); + LOG.debug("Loaded the WSCE native library"); + nativeCodeLoaded = true; + } catch (Throwable t) { + // Ignore failure to load + if(LOG.isDebugEnabled()) { + LOG.debug("Failed to WSCE native library with error: " + t); + LOG.debug("java.library.path=" + + System.getProperty("java.library.path")); + } + } + + if (!nativeCodeLoaded) { + LOG.warn("Unable to load WSCE native library. Windows Secure Container Executor will not work."); + } + } + } - private class WindowsSecureWrapperScriptBuilder - extends LocalWrapperScriptBuilder { - - public WindowsSecureWrapperScriptBuilder(Path containerWorkDir) { - super(containerWorkDir); + /** + * Wraps a process started by the wsce-server service helper. + * + */ + public static class ProcessStub extends Process { + + private final long hProcess; + private final long hThread; + private boolean disposed = false; + + private final InputStream stdErr; + private final InputStream stdOut; + private final OutputStream stdIn; + + public ProcessStub(long hProcess, long hThread, long hStdIn, long hStdOut, long hStdErr) throws IOException { + this.hProcess = hProcess; + this.hThread = hThread; + + this.stdIn = new FileOutputStream(NativeIO.Windows.getFileDescriptorFromHandle(hStdIn)); + this.stdOut = new FileInputStream(NativeIO.Windows.getFileDescriptorFromHandle(hStdOut)); + this.stdErr = new FileInputStream(NativeIO.Windows.getFileDescriptorFromHandle(hStdErr)); } @Override - protected void writeLocalWrapperScript(Path launchDst, Path pidFile, PrintStream pout) { - pout.format("@call \"%s\"", launchDst); + public native void destroy(); + + @Override + public native int exitValue(); + + @Override + public InputStream getErrorStream() { + return stdErr; } + @Override + public InputStream getInputStream() { + return stdOut; + } + @Override + public OutputStream getOutputStream() { + return stdIn; + } + @Override + public native int waitFor() throws InterruptedException; + + public synchronized native void dispose(); + + public native void resume() throws NativeIOException; } - private static class WintuilsProcessStubExecutor implements Shell.ICommandExecutor { - private WinutilsProcessStub processStub; + public synchronized static ProcessStub createTaskAsUser( + String cwd, String jobName, String user, String pidFile, String cmdLine) + throws IOException { + if (!nativeCodeLoaded) { + throw new IOException("WSCE native library is required for createTaskAsUser"); + } + synchronized(Shell.WindowsProcessLaunchLock) { + return createTaskAsUser0(cwd, jobName, user, pidFile, cmdLine); + } + } + + private static native ProcessStub createTaskAsUser0( + String cwd, String jobName, String user, String pidFile, String cmdLine) + throws NativeIOException; + + private static class ProcessStubExecutor implements Shell.ICommandExecutor { + private ProcessStub processStub; private StringBuilder output = new StringBuilder(); private int exitCode; @@ -89,7 +165,7 @@ protected void writeLocalWrapperScript(Path launchDst, Path pidFile, PrintStream private final String cmdLine; private final Configuration conf; - public WintuilsProcessStubExecutor( + public ProcessStubExecutor( Configuration conf, String cwd, String jobName, @@ -161,7 +237,7 @@ public void execute() throws IOException { if (state != State.INIT) { throw new IOException("Process is already started"); } - processStub = NativeIO.createTaskAsUser(cwd, jobName, userName, pidFile, cmdLine); + processStub = createTaskAsUser(cwd, jobName, userName, pidFile, cmdLine); state = State.RUNNING; Thread stdOutReader = startStreamReader(processStub.getInputStream()); @@ -189,6 +265,19 @@ public void dispose() { } } + private class WindowsSecureWrapperScriptBuilder + extends LocalWrapperScriptBuilder { + + public WindowsSecureWrapperScriptBuilder(Path containerWorkDir) { + super(containerWorkDir); + } + + @Override + protected void writeLocalWrapperScript(Path launchDst, Path pidFile, PrintStream pout) { + pout.format("@call \"%s\"", launchDst); + } + } + private String nodeManagerGroup; @Override @@ -283,7 +372,7 @@ public void startLocalizer(Path nmPrivateContainerTokens, String cmdLine = StringUtils.join(command, " "); - WintuilsProcessStubExecutor stubExecutor = new WintuilsProcessStubExecutor( + ProcessStubExecutor stubExecutor = new ProcessStubExecutor( getConf(), cwdApp.getAbsolutePath(), "START_LOCALIZER_" + locId, user, "nul:", cmdLine); @@ -301,7 +390,7 @@ protected ICommandExecutor buildCommandExecutor(String wrapperScriptPath, String String userName, Path pidFile, Configuration conf, File wordDir, Map environment) throws IOException { - return new WintuilsProcessStubExecutor( + return new ProcessStubExecutor( getConf(), wordDir.toString(), containerIdStr, userName, pidFile.toString(), "cmd /c " + wrapperScriptPath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/client.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/client.c new file mode 100644 index 0000000..fc8622a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/client.c @@ -0,0 +1,160 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" +#include +#include +#include "hdpwinutilsvc_h.h" + +#pragma comment(lib, "Rpcrt4.lib") +#pragma comment(lib, "advapi32.lib") + +static ACCESS_MASK CLIENT_MASK = 1; + + +VOID ReportClientError(LPWSTR lpszLocation, DWORD dwError) { + LPWSTR debugMsg = NULL; + int len; + + if (IsDebuggerPresent()) { + len = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, dwError, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR)&debugMsg, 0, NULL); + + LogDebugMessage(L"%s: %s: %x: %.*s\n", GetSystemTimeString(), lpszLocation, dwError, len, debugMsg); + } + + if (NULL != debugMsg) LocalFree(debugMsg); +} + +DWORD RpcCall_TaskCreateAsUser( + LPCWSTR cwd, LPCWSTR jobName, + LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine, + HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr) +{ + DWORD dwError = EXIT_FAILURE; + RPC_STATUS status; + LPWSTR lpszStringBinding = NULL; + ULONG ulCode; + DWORD dwSelfPid = GetCurrentProcessId(); + CREATE_PROCESS_REQUEST request; + CREATE_PROCESS_RESPONSE *response = NULL; + RPC_SECURITY_QOS_V3 qos; + PSID pLocalSystemSid = NULL; + SID_IDENTIFIER_AUTHORITY authNT = SECURITY_NT_AUTHORITY; + + if (!AllocateAndInitializeSid(&authNT, 1, + SECURITY_LOCAL_SYSTEM_RID, + 0, 0, 0, 0, 0, 0, 0, + &pLocalSystemSid)) { + dwError = GetLastError(); + ReportClientError(L"AllocateAndInitializeSid", dwError); + goto done; + } + + ZeroMemory(&qos, sizeof(qos)); + qos.Version = RPC_C_SECURITY_QOS_VERSION_3; + qos.Capabilities = RPC_C_QOS_CAPABILITIES_LOCAL_MA_HINT | RPC_C_QOS_CAPABILITIES_MUTUAL_AUTH; + qos.IdentityTracking = RPC_C_QOS_IDENTITY_DYNAMIC; + qos.ImpersonationType = RPC_C_IMP_LEVEL_DEFAULT; + qos.Sid = pLocalSystemSid; + + ZeroMemory(&request, sizeof(request)); + request.cwd = cwd; + request.jobName = jobName; + request.user = user; + request.pidFile = pidFile; + request.cmdLine = cmdLine; + + status = RpcStringBindingCompose(NULL, + SVCBINDING, + NULL, + SVCNAME, + NULL, + &lpszStringBinding); + if (RPC_S_OK != status) { + ReportClientError(L"RpcStringBindingCompose", status); + dwError = status; + goto done; + } + + status = RpcBindingFromStringBinding(lpszStringBinding, &hHadoopWinutilsSvcBinding); + + if (RPC_S_OK != status) { + ReportClientError(L"RpcBindingFromStringBinding", status); + dwError = status; + goto done; + } + + status = RpcBindingSetAuthInfoEx( + hHadoopWinutilsSvcBinding, + NULL, + RPC_C_AUTHN_LEVEL_PKT_PRIVACY, // AuthnLevel + RPC_C_AUTHN_WINNT, // AuthnSvc + NULL, // AuthnIdentity (self) + RPC_C_AUTHZ_NONE, // AuthzSvc + &qos); + if (RPC_S_OK != status) { + ReportClientError(L"RpcBindingSetAuthInfoEx", status); + dwError = status; + goto done; + } + + RpcTryExcept { + dwError = WinutilsCreateProcessAsUser(dwSelfPid, &request, &response); + } + RpcExcept(1) { + ulCode = RpcExceptionCode(); + ReportClientError(L"RpcExcept", ulCode); + dwError = (DWORD) ulCode; + } + RpcEndExcept; + + if (ERROR_SUCCESS == dwError) { + *phProcess = response->hProcess; + *phThread = response->hThread; + *phStdIn = response->hStdIn; + *phStdOut = response->hStdOut; + *phStdErr = response->hStdErr; + } + + // From here on forward we do no change dwError even on RPC cleanup errors + status = RpcBindingFree(&hHadoopWinutilsSvcBinding); + if (RPC_S_OK != status) { + ReportClientError(L"RpcBindingFree", status); + goto done; + } + +done: + if (pLocalSystemSid) FreeSid(pLocalSystemSid); + + if (NULL != response) { + MIDL_user_free(response); + } + + if (NULL != lpszStringBinding) { + status = RpcStringFree(&lpszStringBinding); + if (RPC_S_OK != status) { + ReportClientError(L"RpcStringFree", status); + } + } + + return dwError; +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/config.cpp b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/config.cpp new file mode 100644 index 0000000..a8d4ade --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/config.cpp @@ -0,0 +1,122 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" +#include +#import "msxml6.dll" + + +#define ERROR_CHECK_HRESULT_DONE(hr, message) \ + if (FAILED(hr)) { \ + dwError = (DWORD) hr; \ + LogDebugMessage(L"%s: %x", message, hr); \ + goto done; \ + } + +DWORD GetConfigValue(__in LPCWSTR keyName, + __out size_t* len, __out_bcount(len) LPCWSTR* value) { + + DWORD dwError = ERROR_SUCCESS; + WCHAR xmlPath[MAX_PATH]; + + *len = 0; + *value = NULL; + + dwError = GetAbsolutePathFromModule(WSCE_CONF_DIR L"\\" WSCE_CONF_FILE, MAX_PATH, xmlPath); + if (dwError) { + goto done; + } + + dwError = GetConfigValueFromXmlFile(xmlPath, keyName, len, value); + if (*len) { + goto done; + } + +done: + if (*len) { + LogDebugMessage(L"GetConfigValue:%d key:%s len:%d value:%.*s from:%s\n", dwError, keyName, *len, *len, *value, xmlPath); + } + return dwError; +} + + +DWORD GetConfigValueFromXmlFile(__in LPCWSTR xmlFile, __in LPCWSTR keyName, + __out size_t* outLen, __out_bcount(len) LPCWSTR* outValue) { + + DWORD dwError = ERROR_SUCCESS; + HRESULT hr; + WCHAR keyXsl[8192]; + size_t len = 0; + LPWSTR value = NULL; + + *outLen = 0; + *outValue = NULL; + + hr = StringCbPrintf(keyXsl, sizeof(keyXsl), L"//configuration/property[name='%s']/value/text()", keyName); + ERROR_CHECK_HRESULT_DONE(hr, L"StringCbPrintf"); + + hr = CoInitialize(NULL); + ERROR_CHECK_HRESULT_DONE(hr, L"CoInitialize"); + + try { + MSXML2::IXMLDOMDocument2Ptr pDoc; + hr = pDoc.CreateInstance(__uuidof(MSXML2::DOMDocument60), NULL, CLSCTX_INPROC_SERVER); + ERROR_CHECK_HRESULT_DONE(hr, L"CreateInstance"); + + pDoc->async = VARIANT_FALSE; + pDoc->validateOnParse = VARIANT_FALSE; + pDoc->resolveExternals = VARIANT_FALSE; + + _variant_t file(xmlFile); + + if (VARIANT_FALSE == pDoc->load(file)) { + dwError = pDoc->parseError->errorCode; + LogDebugMessage(L"load %s failed:%d %s\n", xmlFile, dwError, + static_cast(pDoc->parseError->Getreason())); + goto done; + } + + MSXML2::IXMLDOMElementPtr pRoot = pDoc->documentElement; + MSXML2::IXMLDOMNodePtr keyNode = pRoot->selectSingleNode(keyXsl); + + if (keyNode) { + _bstr_t bstrValue = static_cast<_bstr_t>(keyNode->nodeValue); + len = bstrValue.length(); + value = (LPWSTR) LocalAlloc(LPTR, (len+1) * sizeof(WCHAR)); + LPCWSTR lpwszValue = static_cast(bstrValue); + memcpy(value, lpwszValue, (len) * sizeof(WCHAR)); + LogDebugMessage(L"key:%s :%.*s [%s]\n", keyName, len, value, lpwszValue); + *outLen = len; + *outValue = value; + } + else { + LogDebugMessage(L"node Xpath:%s not found in:%s\n", keyXsl, xmlFile); + } + } + catch(_com_error errorObject) { + dwError = errorObject.Error(); + LogDebugMessage(L"catch _com_error:%x %s\n", dwError, errorObject.ErrorMessage()); + goto done; + } + +done: + CoUninitialize(); + + return dwError; +} + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/hdpwinutilsvc.idl b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/hdpwinutilsvc.idl new file mode 100644 index 0000000..2285178 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/hdpwinutilsvc.idl @@ -0,0 +1,35 @@ +import "oaidl.idl"; +import "ocidl.idl"; + +[ + uuid(0492311C-1718-4F53-A6EB-86AD7039988D), + version(1.0), + pointer_default(unique), + implicit_handle(handle_t hHadoopWinutilsSvcBinding), + endpoint("ncalrpc:[hadoopwinutilsvc]"), +] +interface HadoopWinutilSvc +{ + typedef struct { + [string] const wchar_t* cwd; + [string] const wchar_t* jobName; + [string] const wchar_t* user; + [string] const wchar_t* pidFile; + [string] const wchar_t* cmdLine; + } CREATE_PROCESS_REQUEST; + + typedef struct { + LONG_PTR hProcess; + LONG_PTR hThread; + LONG_PTR hStdIn; + LONG_PTR hStdOut; + LONG_PTR hStdErr; + } CREATE_PROCESS_RESPONSE; + + + error_status_t WinutilsCreateProcessAsUser( + [in] int nmPid, + [in] CREATE_PROCESS_REQUEST *request, + [out] CREATE_PROCESS_RESPONSE **response); + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/org/apache/hadoop/yarn/server/nodemanager/wsce-native.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/org/apache/hadoop/yarn/server/nodemanager/wsce-native.c new file mode 100644 index 0000000..5021c54 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/org/apache/hadoop/yarn/server/nodemanager/wsce-native.c @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#define wsce_process_stub_CLASS "org/apache/hadoop/io/nativeio/NativeIO$WinutilsProcessStub" + + +// class of org.apache.hadoop.io.nativeio.NativeIO.WinutilsProcessStub +static jclass wps_class = NULL; + + +static jmethodID wps_constructor = NULL; +static jfieldID wps_hProcess = NULL; +static jfieldID wps_hThread = NULL; +static jfieldID wps_disposed = NULL; + +// the NativeIOException class and its constructor +static jclass nioe_clazz; +static jmethodID nioe_ctor; + +static void nioe_init(JNIEnv *env) { + // Init NativeIOException + nioe_clazz = (*env)->FindClass( + env, "org/apache/hadoop/io/nativeio/NativeIOException"); + PASS_EXCEPTIONS(env); + + nioe_clazz = (*env)->NewGlobalRef(env, nioe_clazz); + + nioe_ctor = (*env)->GetMethodID(env, nioe_clazz, "", + "(Ljava/lang/String;I)V"); +} + +static void nioe_deinit(JNIEnv *env) { + if (nioe_clazz != NULL) { + (*env)->DeleteGlobalRef(env, nioe_clazz); + nioe_clazz = NULL; + } + nioe_ctor = NULL; +} + + +/* + * Throw a java.IO.IOException, generating the message from errno. + */ +void throw_ioe(JNIEnv* env, int errnum) +{ + + DWORD len = 0; + LPWSTR buffer = NULL; + const jchar* message = NULL; + jstring jstr_message = NULL; + jthrowable obj = NULL; + + len = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, *(DWORD*) (&errnum), // reinterpret cast + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR) &buffer, 0, NULL); + + if (len > 0) + { + message = (const jchar*) buffer; + } + else + { + message = (const jchar*) L"Unknown error."; + } + + if ((jstr_message = (*env)->NewString(env, message, len)) == NULL) + goto err; + LocalFree(buffer); + buffer = NULL; // Set buffer to NULL to avoid double free + + obj = (jthrowable)(*env)->NewObject(env, nioe_clazz, nioe_ctor, + jstr_message, errnum); + if (obj == NULL) goto err; + + (*env)->Throw(env, obj); + return; + +err: + if (jstr_message != NULL) + (*env)->ReleaseStringChars(env, jstr_message, message); + LocalFree(buffer); + return; +} + + +void wsce_process_stub_init(JNIEnv *env) { + if (wps_class != NULL) return; // already initted + + wps_class = (*env)->FindClass(env, wsce_process_stub_CLASS); + PASS_EXCEPTIONS(env); + wps_class = (*env)->NewGlobalRef(env, wps_class); + + wps_hProcess = (*env)->GetFieldID(env, wps_class, "hProcess", "J"); + PASS_EXCEPTIONS(env); + + wps_hThread = (*env)->GetFieldID(env, wps_class, "hThread", "J"); + PASS_EXCEPTIONS(env); + + wps_disposed = (*env)->GetFieldID(env, wps_class, "disposed", "Z"); + PASS_EXCEPTIONS(env); + + wps_constructor = (*env)->GetMethodID(env, wps_class, "", "(JJJJJ)V"); + + LogDebugMessage(L"wsce_process_stub_init\n"); +} + +void wsce_process_stub_deinit(JNIEnv *env) { + if (wps_class != NULL) { + (*env)->DeleteGlobalRef(env, wps_class); + wps_class = NULL; + } + wps_hProcess = NULL; + wps_hThread = NULL; + wps_disposed = NULL; + wps_constructor = NULL; + LogDebugMessage(L"wsce_process_stub_deinit\n"); +} + +jobject wsce_process_stub_create(JNIEnv *env, + jlong hProcess, jlong hThread, jlong hStdIn, jlong hStdOut, jlong hStdErr) { + jobject obj = (*env)->NewObject(env, wps_class, wps_constructor, + hProcess, hThread, hStdIn, hStdOut, hStdErr); + PASS_EXCEPTIONS_RET(env, NULL); + + LogDebugMessage(L"wsce_process_stub_create: %p\n", obj); + + return obj; +} + + +/* + * native void destroy(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT void JNICALL +Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024WinutilsProcessStub_destroy( + JNIEnv *env, jobject objSelf) { + + HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); + LogDebugMessage(L"TerminateProcess: %x\n", hProcess); + TerminateProcess(hProcess, EXIT_FAILURE); +} + +/* + * native void waitFor(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT void JNICALL +Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024WinutilsProcessStub_waitFor( + JNIEnv *env, jobject objSelf) { + + HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); + LogDebugMessage(L"WaitForSingleObject: %x\n", hProcess); + WaitForSingleObject(hProcess, INFINITE); +} + + + +/* + * native void resume(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT void JNICALL +Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024WinutilsProcessStub_resume( + JNIEnv *env, jobject objSelf) { + + DWORD dwError; + HANDLE hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread); + if (-1 == ResumeThread(hThread)) { + dwError = GetLastError(); + LogDebugMessage(L"ResumeThread: %x error:%d\n", hThread, dwError); + throw_ioe(env, dwError); + } +} + +/* + * native int exitValue(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT jint JNICALL +Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_NativeIO_00024WinutilsProcessStub_exitValue( + JNIEnv *env, jobject objSelf) { + + DWORD exitCode; + HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); + GetExitCodeProcess(hProcess, &exitCode); + LogDebugMessage(L"GetExitCodeProcess: %x :%d\n", hProcess, exitCode); + + return exitCode; +} + + +/* + * native void dispose(); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT void JNICALL +Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_NativeIO_00024WinutilsProcessStub_dispose( + JNIEnv *env, jobject objSelf) { + + HANDLE hProcess, hThread; + + jboolean disposed = (*env)->GetBooleanField(env, objSelf, wps_disposed); + + if (JNI_TRUE != disposed) { + hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess); + hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread); + + CloseHandle(hProcess); + CloseHandle(hThread); + (*env)->SetBooleanField(env, objSelf, wps_disposed, JNI_TRUE); + LogDebugMessage(L"disposed: %p\n", objSelf); + } +} + + +/* + * Class: org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor + * Method: createTaskAsUser + * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String)Lorg/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor$ProcessStub + */ +JNIEXPORT jobject JNICALL +Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_createTaskAsUser0(JNIEnv* env, + jclass clazz, jstring cwd, jstring jobName, jstring user, jstring pidFile, jstring cmdLine) { +#ifdef UNIX + THROW(env, "java/io/IOException", + "The function createTaskAsUser is not supported on Unix"); + return -1; +#endif + +#ifdef WINDOWS + LPCWSTR lpszCwd = NULL, lpszJobName = NULL, + lpszUser = NULL, lpszPidFile = NULL, lpszCmdLine = NULL; + DWORD dwError = ERROR_SUCCESS; + HANDLE hProcess = INVALID_HANDLE_VALUE, + hThread = INVALID_HANDLE_VALUE, + hStdIn = INVALID_HANDLE_VALUE, + hStdOut = INVALID_HANDLE_VALUE, + hStdErr = INVALID_HANDLE_VALUE; + jobject ret = NULL; + + lpszCwd = (LPCWSTR) (*env)->GetStringChars(env, cwd, NULL); + if (!lpszCwd) goto done; // exception was thrown + + lpszJobName = (LPCWSTR) (*env)->GetStringChars(env, jobName, NULL); + if (!lpszJobName) goto done; // exception was thrown + + lpszUser = (LPCWSTR) (*env)->GetStringChars(env, user, NULL); + if (!lpszUser) goto done; // exception was thrown + + lpszPidFile = (LPCWSTR) (*env)->GetStringChars(env, pidFile, NULL); + if (!lpszPidFile) goto done; // exception was thrown + + lpszCmdLine = (LPCWSTR) (*env)->GetStringChars(env, cmdLine, NULL); + if (!lpszCmdLine) goto done; // exception was thrown + + LogDebugMessage(L"createTaskAsUser: cwd:%s job:%s user:%s pid:%s cmd:%s\n", + lpszCwd, lpszJobName, lpszUser, lpszPidFile, lpszCmdLine); + + dwError = RpcCall_TaskCreateAsUser(lpszCwd, lpszJobName, lpszUser, lpszPidFile, lpszCmdLine, + &hProcess, &hThread, &hStdIn, &hStdOut, &hStdErr); + + if (ERROR_SUCCESS == dwError) { + ret = wsce_process_stub_create(env, (jlong) hProcess, (jlong) hThread, + (jlong) hStdIn, (jlong) hStdOut, (jlong) hStdErr); + + if (NULL == ret) { + TerminateProcess(hProcess, EXIT_FAILURE); + CloseHandle(hThread); + CloseHandle(hProcess); + CloseHandle(hStdIn); + CloseHandle(hStdOut); + CloseHandle(hStdErr); + } + } + +done: + + if (lpszCwd) (*env)->ReleaseStringChars(env, cwd, lpszCwd); + if (lpszJobName) (*env)->ReleaseStringChars(env, jobName, lpszJobName); + if (lpszUser) (*env)->ReleaseStringChars(env, user, lpszUser); + if (lpszPidFile) (*env)->ReleaseStringChars(env, pidFile, lpszPidFile); + if (lpszCmdLine) (*env)->ReleaseStringChars(env, cmdLine, lpszCmdLine); + + if (dwError != ERROR_SUCCESS) { + throw_ioe (env, dwError); + } + + return ret; + +#endif +} + +/** + * vim: sw=2: ts=2: et: + */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/service.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/service.c new file mode 100644 index 0000000..504f038 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/service.c @@ -0,0 +1,885 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" +#include "wsce-server_msg.h" +#include +#include +#include +#include +#include +#include +#include "hdpwinutilsvc_h.h" + +#pragma comment(lib, "Rpcrt4.lib") +#pragma comment(lib, "advapi32.lib") +#pragma comment(lib, "authz.lib") + +extern DWORD GetConfigValue( + __in LPCWSTR keyName, + __out size_t* len, + __out_bcount(len) LPCWSTR* value); + +#define NM_WSCE_ALLOWED L"yarn.nodemanager.windows-secure-container-executor.allowed" +#define NM_WSCE_DENIED L"yarn.nodemanager.windows-secure-container-executor.denied" + +#define SERVICE_ACCESS_MASK 0x00000001 + +SERVICE_STATUS gSvcStatus; +SERVICE_STATUS_HANDLE gSvcStatusHandle; +HANDLE ghSvcStopEvent = INVALID_HANDLE_VALUE; +HANDLE ghWaitObject = INVALID_HANDLE_VALUE; +HANDLE ghEventLog = INVALID_HANDLE_VALUE; +BOOL isListenning = FALSE; +PSECURITY_DESCRIPTOR pAllowedSD = NULL; + +VOID SvcError(DWORD dwError); +VOID WINAPI SvcMain(DWORD dwArg, LPTSTR* lpszArgv); +DWORD SvcInit(); +DWORD RpcInit(); +DWORD AuthInit(); +VOID ReportSvcStatus( DWORD dwCurrentState, + DWORD dwWin32ExitCode, + DWORD dwWaitHint); +VOID WINAPI SvcCtrlHandler( DWORD dwCtrl ); +VOID CALLBACK SvcShutdown( + _In_ PVOID lpParameter, + _In_ BOOLEAN TimerOrWaitFired); + +#define CHECK_ERROR_DONE(status, expected, category, message) \ + if (status != expected) { \ + ReportSvcCheckError( \ + EVENTLOG_ERROR_TYPE, \ + category, \ + status, \ + message); \ + goto done; \ + } else { \ + LogDebugMessage(L"%s: OK\n", message); \ + } + + +#define CHECK_RPC_STATUS_DONE(status, message) \ + CHECK_ERROR_DONE(status, RPC_S_OK, SERVICE_CATEGORY, message) + +#define CHECK_SVC_STATUS_DONE(status, message) \ + CHECK_ERROR_DONE(status, ERROR_SUCCESS, SERVICE_CATEGORY, message) + +#define CHECK_UNWIND_RPC(rpcCall) { \ + unwindStatus = rpcCall; \ + if (RPC_S_OK != unwindStatus) { \ + ReportSvcCheckError( \ + EVENTLOG_WARNING_TYPE, \ + SERVICE_CATEGORY, \ + unwindStatus, \ + L#rpcCall); \ + } \ + } + + +//---------------------------------------------------------------------------- +// Function: ReportSvcCheckError +// +// Description: +// Reports an error with the system event log and to debugger console (if present) +// +void ReportSvcCheckError(WORD type, WORD category, DWORD dwError, LPCWSTR message) { + int len; + LPWSTR systemMsg = NULL; + LPWSTR appMsg = NULL; + DWORD dwReportError; + LPWSTR reportMsg = NULL; + WCHAR hexError[32]; + LPCWSTR inserts[] = {message, NULL, NULL, NULL}; + HRESULT hr; + + hr = StringCbPrintf(hexError, sizeof(hexError), TEXT("%x"), dwError); + if (SUCCEEDED(hr)) { + inserts[1] = hexError; + } + else { + inserts[1] = L"(Failed to format dwError as string)"; + } + + len = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, dwError, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR)&systemMsg, 0, NULL); + + if (len) { + inserts[2] = systemMsg; + } + else { + inserts[2] = L"(Failed to get the system error message)"; + } + + LogDebugMessage(L"%s:%d %.*s\n", message, dwError, len, systemMsg); + + if (INVALID_HANDLE_VALUE != ghEventLog) { + if (!ReportEvent(ghEventLog, type, category, MSG_CHECK_ERROR, + NULL, // lpUserSid + (WORD) 3, // wNumStrings + (DWORD) 0, // dwDataSize + inserts, // *lpStrings + NULL // lpRawData + )) { + // We tried to report and failed. Send to dbg. + dwReportError = GetLastError(); + len = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, dwReportError, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR)&reportMsg, 0, NULL); + LogDebugMessage(L"ReportEvent: Error:%d %.*s\n", dwReportError, reportMsg); + } + }; + + if (NULL != systemMsg) LocalFree(systemMsg); + if (NULL != reportMsg) LocalFree(reportMsg); +} + + +VOID ReportSvcMessage(WORD type, WORD category, DWORD msgId) { + DWORD dwError; + + if (INVALID_HANDLE_VALUE != ghEventLog) { + if (!ReportEvent(ghEventLog, type, category, msgId, + NULL, // lpUserSid + (WORD) 0, // wNumStrings + (DWORD) 0, // dwDataSize + NULL, // *lpStrings + NULL // lpRawData + )) { + // We tried to report and failed but debugger is attached. Send to dbg. + dwError = GetLastError(); + LogDebugMessage(L"ReportEvent: error %d\n", dwError); + } + } +} + + +//---------------------------------------------------------------------------- +// Function: RunService +// +// Description: +// Registers with NT SCM and starts the service +// +// Returns: +// ERROR_SUCCESS: On success +// Error code otherwise: otherwise +DWORD RunService(__in int argc, __in_ecount(argc) wchar_t *argv[]) +{ + DWORD dwError= ERROR_SUCCESS; + int argStart = 1; + + static const SERVICE_TABLE_ENTRY serviceTable[] = { + { SVCNAME, (LPSERVICE_MAIN_FUNCTION) SvcMain }, + { NULL, NULL } + }; + + dwError = AuthInit(); + if (ERROR_SUCCESS != dwError) { + SvcError(dwError); + goto done; + } + + ghEventLog = RegisterEventSource(NULL, SVCNAME); + if (NULL == ghEventLog) { + dwError = GetLastError(); + CHECK_SVC_STATUS_DONE(dwError, L"RegisterEventSource") + } + + if (!StartServiceCtrlDispatcher(serviceTable)) { + dwError = GetLastError(); + CHECK_SVC_STATUS_DONE(dwError, L"StartServiceCtrlDispatcher") + } + +done: + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: SvcMain +// +// Description: +// Service main entry point. +// +VOID WINAPI SvcMain() { + DWORD dwError = ERROR_SUCCESS; + + gSvcStatusHandle = RegisterServiceCtrlHandler( + SVCNAME, + SvcCtrlHandler); + if( !gSvcStatusHandle ) { + dwError = GetLastError(); + CHECK_SVC_STATUS_DONE(dwError, L"RegisterServiceCtrlHandler") + } + + // These SERVICE_STATUS members remain as set here + gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + gSvcStatus.dwServiceSpecificExitCode = 0; + + // Report initial status to the SCM + ReportSvcStatus( SERVICE_START_PENDING, NO_ERROR, 3000 ); + + // Perform service-specific initialization and work. + dwError = SvcInit(); + +done: + return; +} + +//---------------------------------------------------------------------------- +// Function: SvcInit +// +// Description: +// Initializes the service. +// +DWORD SvcInit() { + DWORD dwError = ERROR_SUCCESS; + + dwError = EnablePrivilege(SE_DEBUG_NAME); + if( dwError != ERROR_SUCCESS ) { + goto done; + } + + // The recommended way to shutdown the service is to use an event + // and attach a callback with RegisterWaitForSingleObject + // + ghSvcStopEvent = CreateEvent( + NULL, // default security attributes + TRUE, // manual reset event + FALSE, // not signaled + NULL); // no name + + if ( ghSvcStopEvent == NULL) + { + dwError = GetLastError(); + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + dwError, L"CreateEvent"); + ReportSvcStatus( SERVICE_STOPPED, dwError, 0 ); + goto done; + } + + if (!RegisterWaitForSingleObject (&ghWaitObject, + ghSvcStopEvent, + SvcShutdown, + NULL, + INFINITE, + WT_EXECUTEONLYONCE)) { + dwError = GetLastError(); + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + dwError, L"RegisterWaitForSingleObject"); + CloseHandle(ghSvcStopEvent); + ReportSvcStatus( SERVICE_STOPPED, dwError, 0 ); + goto done; + } + + // Report running status when initialization is complete. + ReportSvcStatus( SERVICE_RUNNING, NO_ERROR, 0 ); + + dwError = RpcInit(); + +done: + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: RpcAuthorizeCallback +// +// Description: +// RPC Authorization callback. +// +// Returns: +// RPC_S_OK for access authorized +// RPC_S_ACCESS_DENIED for access denied +// +RPC_STATUS CALLBACK RpcAuthorizeCallback ( + RPC_IF_HANDLE hInterface, + void* pContext) +{ + RPC_STATUS status, + unwindStatus, + authStatus = RPC_S_ACCESS_DENIED; + DWORD dwError; + LUID luidReserved2; + AUTHZ_ACCESS_REQUEST request; + AUTHZ_ACCESS_REPLY reply; + AUTHZ_CLIENT_CONTEXT_HANDLE hClientContext = NULL; + DWORD authError = ERROR_SUCCESS; + DWORD saclResult = 0; + ACCESS_MASK grantedMask = 0; + + ZeroMemory(&luidReserved2, sizeof(luidReserved2)); + ZeroMemory(&request, sizeof(request)); + ZeroMemory(&reply, sizeof(reply)); + + status = RpcGetAuthorizationContextForClient(NULL, + FALSE, // ImpersonateOnReturn + NULL, // Reserved1 + NULL, // pExpirationTime + luidReserved2, // Reserved2 + 0, // Reserved3 + NULL, // Reserved4 + &hClientContext); + CHECK_RPC_STATUS_DONE(status, L"RpcGetAuthorizationContextForClient"); + + request.DesiredAccess = MAXIMUM_ALLOWED; + reply.Error = &authError; + reply.SaclEvaluationResults = &saclResult; + reply.ResultListLength = 1; + reply.GrantedAccessMask = &grantedMask; + + if (!AuthzAccessCheck( + 0, + hClientContext, + &request, + NULL, // AuditEvent + pAllowedSD, + NULL, // OptionalSecurityDescriptorArray + 0, // OptionalSecurityDescriptorCount + &reply, + NULL // phAccessCheckResults + )) { + dwError = GetLastError(); + CHECK_SVC_STATUS_DONE(dwError, L"AuthzAccessCheck"); + } + + LogDebugMessage(L"AutzAccessCheck: Error:%d sacl:%d access:%d\n", + authError, saclResult, grantedMask); + if (authError == ERROR_SUCCESS && (grantedMask & SERVICE_ACCESS_MASK)) { + authStatus = RPC_S_OK; + } + +done: + if (NULL != hClientContext) CHECK_UNWIND_RPC(RpcFreeAuthorizationContext(&hClientContext)); + return authStatus; +} + +//---------------------------------------------------------------------------- +// Function: AuthInit +// +// Description: +// Initializes the authorization structures (security descriptor). +// +// Notes: +// This is called from RunService solely for debugging purposed +// so that it can be tested by wimply running winutil service from CLI (no SCM) +// +DWORD AuthInit() { + DWORD dwError = ERROR_SUCCESS; + int count = 0; + int crt = 0; + int len = 0; + LPCWSTR value = NULL; + WCHAR** tokens = NULL; + LPWSTR lpszSD = NULL; + ULONG cchSD = 0; + DWORD dwBufferSize = 0; + int allowedCount = 0; + PSID* allowedSids = NULL; + + + dwError = GetConfigValue(NM_WSCE_ALLOWED, &len, &value); + CHECK_SVC_STATUS_DONE(dwError, L"GetConfigValue"); + + if (0 == len) { + CHECK_SVC_STATUS_DONE(ERROR_BAD_CONFIGURATION, NM_WSCE_ALLOWED); + } + + dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens); + CHECK_SVC_STATUS_DONE(dwError, L"SplitStringIgnoreSpaceW"); + + allowedSids = (PSID*) LocalAlloc(LPTR, sizeof(PSID) * count); + for (crt = 0; crt < count; ++crt) { + dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]); + CHECK_SVC_STATUS_DONE(dwError, L"GetSidFromAcctNameW"); + } + + allowedCount = count; + + dwError = BuildServiceSecurityDescriptor(SERVICE_ACCESS_MASK, + allowedCount, allowedSids, 0, NULL, &pAllowedSD); + CHECK_SVC_STATUS_DONE(dwError, L"BuildServiceSecurityDescriptor"); + +done: + if (lpszSD) LocalFree(lpszSD); + if (value) LocalFree(value); + if (tokens) LocalFree(tokens); + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: RpcInit +// +// Description: +// Initializes the RPC infrastructure and starts the RPC listenner. +// +DWORD RpcInit() { + RPC_STATUS status; + DWORD dwError; + + status = RpcServerUseProtseqIf(SVCBINDING, + RPC_C_LISTEN_MAX_CALLS_DEFAULT, + HadoopWinutilSvc_v1_0_s_ifspec, + NULL); + if (RPC_S_OK != status) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + status, L"RpcServerUseProtseqIf"); + SvcError(status); + dwError = status; + goto done; + } + + status = RpcServerRegisterIfEx(HadoopWinutilSvc_v1_0_s_ifspec, + NULL, // MgrTypeUuid + NULL, // MgrEpv + RPC_IF_ALLOW_LOCAL_ONLY, // Flags + RPC_C_LISTEN_MAX_CALLS_DEFAULT, // Max calls + RpcAuthorizeCallback); // Auth callback + + if (RPC_S_OK != status) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + status, L"RpcServerRegisterIfEx"); + SvcError(status); + dwError = status; + goto done; + } + + status = RpcServerListen(1, RPC_C_LISTEN_MAX_CALLS_DEFAULT, TRUE); + if (RPC_S_ALREADY_LISTENING == status) { + ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY, + status, L"RpcServerListen"); + } + else if (RPC_S_OK != status) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + status, L"RpcServerListen"); + SvcError(status); + dwError = status; + goto done; + } + + isListenning = TRUE; + + ReportSvcMessage(EVENTLOG_INFORMATION_TYPE, SERVICE_CATEGORY, + MSG_RPC_SERVICE_HAS_STARTED); + +done: + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: RpcStop +// +// Description: +// Tears down the RPC infrastructure and stops the RPC listenner. +// +VOID RpcStop() { + RPC_STATUS status; + + if (isListenning) { + + status = RpcMgmtStopServerListening(NULL); + isListenning = FALSE; + + if (RPC_S_OK != status) { + ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY, + status, L"RpcMgmtStopServerListening"); + } + + ReportSvcMessage(EVENTLOG_INFORMATION_TYPE, SERVICE_CATEGORY, + MSG_RPC_SERVICE_HAS_STOPPED); + } +} + +//---------------------------------------------------------------------------- +// Function: CleanupHandles +// +// Description: +// Cleans up the global service handles. +// +VOID CleanupHandles() { + if (INVALID_HANDLE_VALUE != ghWaitObject) { + UnregisterWait(ghWaitObject); + ghWaitObject = INVALID_HANDLE_VALUE; + } + if (INVALID_HANDLE_VALUE != ghSvcStopEvent) { + CloseHandle(ghSvcStopEvent); + ghSvcStopEvent = INVALID_HANDLE_VALUE; + } + if (INVALID_HANDLE_VALUE != ghEventLog) { + DeregisterEventSource(ghEventLog); + ghEventLog = INVALID_HANDLE_VALUE; + } +} + +//---------------------------------------------------------------------------- +// Function: SvcError +// +// Description: +// Aborts the startup sequence. Reports error, stops RPC, cleans up globals. +// +VOID SvcError(DWORD dwError) { + RpcStop(); + CleanupHandles(); + ReportSvcStatus( SERVICE_STOPPED, dwError, 0 ); +} + +//---------------------------------------------------------------------------- +// Function: SvcShutdown +// +// Description: +// Callback when the shutdown event is signaled. Stops RPC, cleans up globals. +// +VOID CALLBACK SvcShutdown( + _In_ PVOID lpParameter, + _In_ BOOLEAN TimerOrWaitFired) { + RpcStop(); + CleanupHandles(); + ReportSvcStatus( SERVICE_STOPPED, NO_ERROR, 0 ); +} + +//---------------------------------------------------------------------------- +// Function: SvcCtrlHandler +// +// Description: +// Callback from SCM for for service events (signals). +// +// Notes: +// Shutdown is indirect, we set her the STOP_PENDING state and signal the stop event. +// Signaling the event invokes SvcShutdown which completes the shutdown. +// This two staged approach allows the SCM handler to complete fast, +// not blocking the SCM big fat global lock. +// +VOID WINAPI SvcCtrlHandler( DWORD dwCtrl ) +{ + // Handle the requested control code. + + switch(dwCtrl) + { + case SERVICE_CONTROL_STOP: + ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0); + + // Signal the service to stop. + SetEvent(ghSvcStopEvent); + + return; + + default: + break; + } + +} + +//---------------------------------------------------------------------------- +// Function: ReportSvcStatus +// +// Description: +// Updates the service status with the SCM. +// +VOID ReportSvcStatus( DWORD dwCurrentState, + DWORD dwWin32ExitCode, + DWORD dwWaitHint) +{ + static DWORD dwCheckPoint = 1; + DWORD dwError; + + // Fill in the SERVICE_STATUS structure. + + gSvcStatus.dwCurrentState = dwCurrentState; + gSvcStatus.dwWin32ExitCode = dwWin32ExitCode; + gSvcStatus.dwWaitHint = dwWaitHint; + + if (dwCurrentState == SERVICE_START_PENDING) + gSvcStatus.dwControlsAccepted = 0; + else gSvcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP; + + if ( (dwCurrentState == SERVICE_RUNNING) || + (dwCurrentState == SERVICE_STOPPED) ) + gSvcStatus.dwCheckPoint = 0; + else gSvcStatus.dwCheckPoint = dwCheckPoint++; + + // Report the status of the service to the SCM. + if (!SetServiceStatus( gSvcStatusHandle, &gSvcStatus)) { + dwError = GetLastError(); + ReportSvcCheckError(EVENTLOG_WARNING_TYPE, SERVICE_CATEGORY, + dwError, L"SetServiceStatus"); + }; +} + +//---------------------------------------------------------------------------- +// Function: WinutilsCreateProcessAsUser +// +// Description: +// The RPC midl declared function implementation +// +// Returns: +// ERROR_SUCCESS: On success +// Error code otherwise: otherwise +// +// Notes: +// This is the entry point when the NodeManager does the RPC call +// Note that the RPC call does not do any S4U work. Is simply spawns (suspended) wintutils +// using the right command line and the handles over the spwaned process to the NM +// The actual S4U work occurs in the spawned process, run and monitored by the NM +// +error_status_t WinutilsCreateProcessAsUser( + /* [in] */ int nmPid, + /* [in] */ CREATE_PROCESS_REQUEST *request, + /* [out] */ CREATE_PROCESS_RESPONSE **response) { + + DWORD dwError = ERROR_SUCCESS; + LPCWSTR inserts[] = {request->cwd, request->jobName, request->user, request->pidFile, request->cmdLine, NULL}; + WCHAR winutilsPath[MAX_PATH]; + WCHAR fullCmdLine[32768]; + HANDLE taskStdInRd = INVALID_HANDLE_VALUE, taskStdInWr = INVALID_HANDLE_VALUE, + taskStdOutRd = INVALID_HANDLE_VALUE, taskStdOutWr = INVALID_HANDLE_VALUE, + taskStdErrRd = INVALID_HANDLE_VALUE, taskStdErrWr = INVALID_HANDLE_VALUE, + hNmProcess = INVALID_HANDLE_VALUE, + hDuplicateProcess = INVALID_HANDLE_VALUE, + hDuplicateThread = INVALID_HANDLE_VALUE, + hDuplicateStdIn = INVALID_HANDLE_VALUE, + hDuplicateStdOut = INVALID_HANDLE_VALUE, + hDuplicateStdErr = INVALID_HANDLE_VALUE, + hSelfProcess = INVALID_HANDLE_VALUE; + BOOL fMustCleanupProcess = FALSE; + + HRESULT hr; + STARTUPINFO si; + PROCESS_INFORMATION pi; + SECURITY_ATTRIBUTES saTaskStdInOutErr; + + ZeroMemory( &si, sizeof(si) ); + si.cb = sizeof(si); + ZeroMemory( &pi, sizeof(pi) ); + pi.hProcess = INVALID_HANDLE_VALUE; + pi.hThread = INVALID_HANDLE_VALUE; + ZeroMemory( &saTaskStdInOutErr, sizeof(saTaskStdInOutErr)); + + // NB: GetCurrentProcess returns a pseudo-handle that just so happens + // has the value -1, ie. INVALID_HANDLE_VALUE. It cannot fail. + // + hSelfProcess = GetCurrentProcess(); + + hNmProcess = OpenProcess(PROCESS_DUP_HANDLE, FALSE, nmPid); + if (NULL == hNmProcess) { + dwError = GetLastError(); + goto done; + } + + dwError = GetAbsolutePathFromModule(WSCE_WINUTILS_BIN, MAX_PATH, winutilsPath); + if (dwError) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + dwError, L"GetAbsolutePathFromModule"); + goto done; + } + + // NB. We can call CreateProcess("winutils","task create ...") or we can call + // CreateProcess(NULL, "winutils task create"). Only the second form passes "task" as + // argv[1], as expected by main. First form passes "task" as argv[0] and main fails. + + hr = StringCbPrintf(fullCmdLine, sizeof(fullCmdLine), L"\"%s\" task createAsUser %ls %ls %ls %ls", + winutilsPath, + request->jobName, request->user, request->pidFile, request->cmdLine); + if (FAILED(hr)) { + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + hr, L"StringCbPrintf:fullCmdLine"); + goto done; + } + + LogDebugMessage(L"[%ls]: %ls\\" WSCE_WINUTILS_BIN L" %ls\n", request->cwd, selfModulePath, fullCmdLine); + + // stdin/stdout/stderr redirection is handled here + // We create 3 anonimous named pipes. + // Security attributes are required so that the handles can be inherited. + // We assign one end of the pipe to the process (stdin gets a read end, stdout gets a write end) + // We then duplicate the other end in the NM process, and we close our own handle + // Finally we return the duplicate handle values to the NM + // The NM will attach Java file dscriptors to the duplicated handles and + // read/write them as ordinary Java InputStream/OutputStream objects + + si.dwFlags |= STARTF_USESTDHANDLES; + + saTaskStdInOutErr.nLength = sizeof(SECURITY_ATTRIBUTES); + saTaskStdInOutErr.bInheritHandle = TRUE; + saTaskStdInOutErr.lpSecurityDescriptor = NULL; + + if (!CreatePipe(&taskStdInRd, &taskStdInWr, &saTaskStdInOutErr, 0)) { + dwError = GetLastError(); + goto done; + } + if (!SetHandleInformation(taskStdInWr, HANDLE_FLAG_INHERIT, FALSE)) { + dwError = GetLastError(); + goto done; + } + si.hStdInput = taskStdInRd; + + if (!CreatePipe(&taskStdOutRd, &taskStdOutWr, &saTaskStdInOutErr, 0)) { + dwError = GetLastError(); + goto done; + } + if (!SetHandleInformation(taskStdOutRd, HANDLE_FLAG_INHERIT, FALSE)) { + dwError = GetLastError(); + goto done; + } + si.hStdOutput = taskStdOutWr; + + if (!CreatePipe(&taskStdErrRd, &taskStdErrWr, &saTaskStdInOutErr, 0)) { + dwError = GetLastError(); + goto done; + } + if (!SetHandleInformation(taskStdErrRd, HANDLE_FLAG_INHERIT, FALSE)) { + dwError = GetLastError(); + goto done; + } + si.hStdError = taskStdErrWr; + + if (!CreateProcess( + NULL, // lpApplicationName, + fullCmdLine, // lpCommandLine, + NULL, // lpProcessAttributes, + NULL, // lpThreadAttributes, + TRUE, // bInheritHandles, + CREATE_SUSPENDED, // dwCreationFlags, + NULL, // lpEnvironment, + request->cwd, // lpCurrentDirectory, + &si, // lpStartupInfo + &pi)) { // lpProcessInformation + + dwError = GetLastError(); + ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY, + dwError, L"CreateProcess"); + goto done; + } + + fMustCleanupProcess = TRUE; + + LogDebugMessage(L"CreateProcess: pid:%x\n", pi.dwProcessId); + + if (!DuplicateHandle(hSelfProcess, pi.hProcess, hNmProcess, + &hDuplicateProcess, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: pi.hProcess\n"); + goto done; + } + + if (!DuplicateHandle(hSelfProcess, pi.hThread, hNmProcess, + &hDuplicateThread, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: pi.hThread\n"); + goto done; + } + + if (!DuplicateHandle(hSelfProcess, taskStdInWr, hNmProcess, + &hDuplicateStdIn, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: taskStdInWr\n"); + goto done; + } + + if (!DuplicateHandle(hSelfProcess, taskStdOutRd, hNmProcess, + &hDuplicateStdOut, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: taskStdOutRd\n"); + goto done; + } + + if (!DuplicateHandle(hSelfProcess, taskStdErrRd, hNmProcess, + &hDuplicateStdErr, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + dwError = GetLastError(); + LogDebugMessage(L"failed: taskStdErrRd\n"); + goto done; + } + + *response = (CREATE_PROCESS_RESPONSE*) MIDL_user_allocate(sizeof(CREATE_PROCESS_RESPONSE)); + if (NULL == *response) { + dwError = ERROR_OUTOFMEMORY; + LogDebugMessage(L"Failed to allocate CREATE_PROCESS_RESPONSE* response\n"); + goto done; + } + + // We're now transfering ownership of the duplicated handles to the caller + // If the RPC call fails *after* this point the handles are leaked inside the NM process + + (*response)->hProcess = hDuplicateProcess; + (*response)->hThread = hDuplicateThread; + (*response)->hStdIn = hDuplicateStdIn; + (*response)->hStdOut = hDuplicateStdOut; + (*response)->hStdErr = hDuplicateStdErr; + + fMustCleanupProcess = FALSE; + +done: + + if (fMustCleanupProcess) { + LogDebugMessage(L"Cleaning process: %d due to error:%d\n", pi.dwProcessId, dwError); + TerminateProcess(pi.hProcess, EXIT_FAILURE); + + // cleanup the duplicate handles inside the NM. + + if (INVALID_HANDLE_VALUE != hDuplicateProcess) { + DuplicateHandle(hNmProcess, hDuplicateProcess, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + if (INVALID_HANDLE_VALUE != hDuplicateThread) { + DuplicateHandle(hNmProcess, hDuplicateThread, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + if (INVALID_HANDLE_VALUE != hDuplicateStdIn) { + DuplicateHandle(hNmProcess, hDuplicateStdIn, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + if (INVALID_HANDLE_VALUE != hDuplicateStdOut) { + DuplicateHandle(hNmProcess, hDuplicateStdOut, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + if (INVALID_HANDLE_VALUE != hDuplicateStdErr) { + DuplicateHandle(hNmProcess, hDuplicateStdErr, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE); + } + } + + if (INVALID_HANDLE_VALUE != hSelfProcess) CloseHandle(hSelfProcess); + if (INVALID_HANDLE_VALUE != hNmProcess) CloseHandle(hNmProcess); + if (INVALID_HANDLE_VALUE != taskStdInRd) CloseHandle(taskStdInRd); + if (INVALID_HANDLE_VALUE != taskStdInWr) CloseHandle(taskStdInWr); + if (INVALID_HANDLE_VALUE != taskStdOutRd) CloseHandle(taskStdOutRd); + if (INVALID_HANDLE_VALUE != taskStdOutWr) CloseHandle(taskStdOutWr); + if (INVALID_HANDLE_VALUE != taskStdErrRd) CloseHandle(taskStdErrRd); + if (INVALID_HANDLE_VALUE != taskStdErrWr) CloseHandle(taskStdErrWr); + + + // This is closing our own process/thread handles. + // If the transfer was succesfull the NM has its own duplicates (if any) + if (INVALID_HANDLE_VALUE != pi.hThread) CloseHandle(pi.hThread); + if (INVALID_HANDLE_VALUE != pi.hProcess) CloseHandle(pi.hProcess); + + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: ServiceUsage +// +// Description: +// Prints the CLI arguments for service command. +// +void ServiceUsage() +{ + fwprintf(stdout, L"\ + Usage: service\n\ + Starts the impersonation helper service.\n\ + This should be called from the SCM.\n\ + The impersonation helper service must run as a high privileged account (LocalSystem)\n\ + and is used by the NodeManager to spawn secure containers.\n"); +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-client.vcxproj b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-client.vcxproj new file mode 100644 index 0000000..0ed1bf6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-client.vcxproj @@ -0,0 +1,109 @@ + + + + + + + + Debug + x64 + + + Release + x64 + + + + {31F3D03C-3787-4657-979E-43E52EFA303B} + wsceclient + + + + DynamicLibrary + true + Unicode + + + DynamicLibrary + false + true + Unicode + + + + + + + + + + + + + .dll + + + .dll + + + + Level3 + Disabled + + + true + + + + + Level3 + MaxSpeed + true + true + WIN32;NDEBUG;_WINDOWS;_USRDLL;NATIVE_EXPORTS;%(PreprocessorDefinitions) + $(winutilsdir)\include;$(winutilsdir)\..\native\src;$(IntermediateOutputPath);%JAVA_HOME%\include;%JAVA_HOME%\include\win32;%(AdditionalIncludeDirectories) + + + true + true + true + libwinutils.lib;%(AdditionalDependencies) + $(winutilsdir)\..\..\..\target\bin;%(AdditionalLibraryDirectories) + + + true + X64 + $(IntermediateOutputPath) + true + true + true + 2 + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-main.c new file mode 100644 index 0000000..164f7c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-main.c @@ -0,0 +1,61 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" + +static void Usage(LPCWSTR program); + +int wmain(__in int argc, __in_ecount(argc) wchar_t* argv[]) +{ + LPCWSTR cmd = NULL; + + if (argc < 2) + { + Usage(argv[0]); + return EXIT_FAILURE; + } + + cmd = argv[1]; + + if (wcscmp(L"service", cmd) == 0) + { + return RunService(argc, argv); + } + else if (wcscmp(L"help", cmd) == 0) + { + Usage(argv[0]); + return EXIT_SUCCESS; + } + else + { + Usage(argv[0]); + return EXIT_FAILURE; + } +} + +static void Usage(LPCWSTR program) +{ + fwprintf(stdout, L"Usage: %s [command] ...\n\ +Provide secure Windows container executor for Hadoop Yarn Nodemanager on Windows.\n\n\ +The available commands and their usages are:\n\n", program); + + fwprintf(stdout, L"%-15s%s\n\n", L"service", L"LRPC Service operations."); + ServiceUsage(); + + fwprintf(stdout, L"\n\n"); +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-server.mc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-server.mc new file mode 100644 index 0000000..a2e30ad --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-server.mc @@ -0,0 +1,46 @@ +; // winutils.mc + +; // EventLog messages for Hadoop winutils service. + + +LanguageNames=(English=0x409:MSG00409) + + +; // The following are the categories of events. + +MessageIdTypedef=WORD + +MessageId=0x1 +SymbolicName=SERVICE_CATEGORY +Language=English +Service Events +. + +MessageId=0x2 +SymbolicName=LOG_CATEGORY +Language=English +Task Events +. + +; // The following are the message definitions. + +MessageIdTypedef=DWORD + +MessageId=0x80 +SymbolicName=MSG_CHECK_ERROR +Language=English +%1. Error %2: %3. +. + +MessageId=0x100 +SymbolicName=MSG_RPC_SERVICE_HAS_STARTED +Language=English +The LPC server is listenning. +. + +MessageId=0x200 +SymbolicName=MSG_RPC_SERVICE_HAS_STOPPED +Language=English +The LPC server has stopped listenning. +. + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-server.vcxproj b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-server.vcxproj new file mode 100644 index 0000000..9e7468f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce-server.vcxproj @@ -0,0 +1,126 @@ + + + + + + + + Debug + x64 + + + Release + x64 + + + + {0581CE10-6130-4C39-9D29-4B3EB980C547} + wsceserver + + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + $(IntermediateOutputPath) + + + Compiling Messages + mc.exe $(TargetName).mc -z $(TargetName)_msg -r $(IntermediateOutputPath) -h $(IntermediateOutputPath) -U + $(IntermediateOutputPath)\$(TargetName)_msg.rc,$(IntermediateOutputPath)\$(TargetName)_msg.h + + + true + X64 + $(IntermediateOutputPath)\idl + true + true + true + 2 + + + + + Level3 + Disabled + + + true + + + + Midl + ClCompile,ResourceCompile + + + + + + + Level3 + MaxSpeed + true + true + NotUsing + WSCE_WINUTILS_BIN=L"$(WsceWinutilsBin)";WSCE_CONF_DIR=L"$(WsceConfDir)";WSCE_CONF_FILE=L"$(WsceConfFile)";%(PreprocessorDefinitions) + WIN32;NDEBUG;_WINDOWS;_CONSOLE;%(PreprocessorDefinitions) + $(winutilsdir)\include;%(AdditionalIncludeDirectories) + + + true + true + true + libwinutils.lib;%(AdditionalDependencies) + $(winutilsdir)\..\..\..\target\bin;%(AdditionalLibraryDirectories) + + + + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce.sln b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce.sln new file mode 100644 index 0000000..e2759fe --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/wsce/wsce.sln @@ -0,0 +1,24 @@ + +Microsoft Visual Studio Solution File, Format Version 11.00 +# Visual Studio 2010 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "wsce-client", "wsce-client.vcxproj", "{31F3D03C-3787-4657-979E-43E52EFA303B}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "wsce-server", "wsce-server.vcxproj", "{0581CE10-6130-4C39-9D29-4B3EB980C547}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {31F3D03C-3787-4657-979E-43E52EFA303B}.Debug|x64.ActiveCfg = Debug|x64 + {31F3D03C-3787-4657-979E-43E52EFA303B}.Release|x64.ActiveCfg = Release|x64 + {31F3D03C-3787-4657-979E-43E52EFA303B}.Release|x64.Build.0 = Release|x64 + {0581CE10-6130-4C39-9D29-4B3EB980C547}.Debug|x64.ActiveCfg = Debug|x64 + {0581CE10-6130-4C39-9D29-4B3EB980C547}.Release|x64.ActiveCfg = Release|x64 + {0581CE10-6130-4C39-9D29-4B3EB980C547}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal