diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index a06e3a6..e401fb0 100644
--- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -268,7 +268,11 @@ private FSDataOutputStream create(Path f, boolean overwrite,
throw new IOException("Mkdirs failed to create " + parent.toString());
}
return new FSDataOutputStream(new BufferedOutputStream(
- new LocalFSFileOutputStream(f, false), bufferSize), statistics);
+ createOutputStream(f, false), bufferSize), statistics);
+ }
+
+ protected OutputStream createOutputStream(Path f, boolean append) throws IOException {
+ return new LocalFSFileOutputStream(f, append);
}
@Override
@@ -406,6 +410,10 @@ public boolean delete(Path p, boolean recursive) throws IOException {
}
return Arrays.copyOf(results, j);
}
+
+ protected boolean mkOneDir(File p2f) throws IOException {
+ return p2f.mkdir();
+ }
/**
* Creates the specified directory hierarchy. Does not
@@ -418,8 +426,9 @@ public boolean mkdirs(Path f) throws IOException {
}
Path parent = f.getParent();
File p2f = pathToFile(f);
+ File parent2f = null;
if(parent != null) {
- File parent2f = pathToFile(parent);
+ parent2f = pathToFile(parent);
if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
throw new ParentNotDirectoryException("Parent path is not a directory: "
+ parent);
@@ -429,8 +438,8 @@ public boolean mkdirs(Path f) throws IOException {
throw new FileNotFoundException("Destination exists" +
" and is not a directory: " + p2f.getCanonicalPath());
}
- return (parent == null || mkdirs(parent)) &&
- (p2f.mkdir() || p2f.isDirectory());
+ return (parent == null || parent2f.exists() || mkdirs(parent)) &&
+ (mkOneDir(p2f) || p2f.isDirectory());
}
@Override
diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index f8e9edf..b897815 100644
--- hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.io.nativeio;
-import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileDescriptor;
import java.io.FileInputStream;
@@ -37,6 +36,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.HardLink;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
@@ -505,6 +505,8 @@ public static native void munmap(long addr, long length)
public static final long FILE_BEGIN = 0;
public static final long FILE_CURRENT = 1;
public static final long FILE_END = 2;
+
+ public static final long FILE_ATTRIBUTE_NORMAL = 0x00000080L;
/** Wrapper around CreateFile() on Windows */
public static native FileDescriptor createFile(String path,
@@ -849,6 +851,72 @@ private static native void renameTo0(String src, String dst)
private static native void link0(String src, String dst)
throws NativeIOException;
+ public static class Elevated {
+ private static final int MOVE_FILE = 1;
+ private static final int COPY_FILE = 2;
+
+ public static void mkdir(Path dirName) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for mkdir");
+ }
+ elevatedMkDirImpl(dirName.toString());
+ }
+
+ private static native void elevatedMkDirImpl(String dirName) throws IOException;
+
+ public static void chown(Path fileName, String user, String group) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for chown");
+ }
+ elevatedChownImpl(fileName.toString(), user, group);
+ }
+
+ private static native void elevatedChownImpl(String fileName, String user, String group) throws IOException;
+
+ public static void move(Path src, Path dst, boolean replaceExisting) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for move");
+ }
+ elevatedCopyImpl(MOVE_FILE, src.toString(), dst.toString(), replaceExisting);
+ }
+
+ public static void copy(Path src, Path dst, boolean replaceExisting) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for copy");
+ }
+ elevatedCopyImpl(COPY_FILE, src.toString(), dst.toString(), replaceExisting);
+ }
+
+ private static native void elevatedCopyImpl(int operation, String src, String dst, boolean replaceExisting) throws IOException;
+
+ public static void chmod(Path fileName, int mode) {
+
+ }
+
+ public static OutputStream create(Path f, boolean append) throws IOException {
+ if (!nativeLoaded) {
+ throw new IOException("NativeIO libraries are required for create");
+ }
+
+ long desiredAccess = Windows.GENERIC_WRITE;
+ long shareMode = 0L;
+ long creationDisposition = append ? Windows.OPEN_ALWAYS : Windows.CREATE_ALWAYS;
+ long flags = Windows.FILE_ATTRIBUTE_NORMAL;
+
+ String fileName = f.toString();
+ fileName = fileName.replace('/', '\\');
+
+ long hFile = elevatedCreateImpl(
+ fileName, desiredAccess, shareMode, creationDisposition, flags);
+ return new FileOutputStream(
+ WinutilsProcessStub.getFileDescriptorFromHandle(hFile));
+ }
+
+ private static native long elevatedCreateImpl(String path, long desiredAccess, long shareMode,
+ long creationDisposition, long flags) throws IOException;
+
+ }
+
/**
* Wraps a process started by the winutils service helper.
*
@@ -872,7 +940,7 @@ public WinutilsProcessStub(long hProcess, long hThread, long hStdIn, long hStdOu
this.stdErr = new FileInputStream(getFileDescriptorFromHandle(hStdErr));
}
- private static native FileDescriptor getFileDescriptorFromHandle(long handle);
+ public static native FileDescriptor getFileDescriptorFromHandle(long handle);
@Override
public native void destroy();
diff --git hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
index df57e0e..ee4db6f 100644
--- hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
+++ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
@@ -219,7 +219,7 @@ static int map_fadvise_flag(jint flag) {
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_nativeio_NativeIO_initNative(
- JNIEnv *env, jclass clazz) {
+ JNIEnv *env, jclass clazz) {
stat_init(env, clazz);
PASS_EXCEPTIONS_GOTO(env, error);
nioe_init(env);
@@ -1236,6 +1236,199 @@ done:
#endif
}
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedChownImpl
+ * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedChownImpl(JNIEnv* env,
+ jclass clazz, jstring jpath, jstring juser, jstring jgroup) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedSetOwner0 is not supported on Unix");
+ return NULL;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR path = NULL, user = NULL, group = NULL;
+ DWORD dwError;
+
+ path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
+ if (!path) goto done; // exception was thrown
+
+ if (juser) {
+ user = (LPCWSTR) (*env)->GetStringChars(env, juser, NULL);
+ if (!user) goto done; // exception was thrown
+ }
+
+ if (jgroup) {
+ group = (LPCWSTR) (*env)->GetStringChars(env, jgroup, NULL);
+ if (!group) goto done; // exception was thrown
+ }
+
+ dwError = RpcCall_WinutilsChown(path, user, group);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (path) (*env)->ReleaseStringChars(env, jpath, path);
+ if (user) (*env)->ReleaseStringChars(env, juser, user);
+ if (group) (*env)->ReleaseStringChars(env, jgroup, group);
+
+#endif
+
+}
+
+
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedMkDirImpl
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedMkDirImpl(JNIEnv* env,
+ jclass clazz, jstring jpath) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedMkDirImpl is not supported on Unix");
+ return NULL;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR path = NULL, user = NULL, group = NULL;
+ DWORD dwError;
+
+ path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
+ if (!path) goto done; // exception was thrown
+
+ dwError = RpcCall_WinutilsMkDir(path);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (path) (*env)->ReleaseStringChars(env, jpath, path);
+
+#endif
+
+}
+
+
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedChmodImpl
+ * Signature: (Ljava/lang/String;I)V
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedChmodImpl(JNIEnv* env,
+ jclass clazz, jstring jpath, jint jmode) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedChmodImpl is not supported on Unix");
+ return NULL;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR path = NULL;
+ DWORD dwError;
+
+ path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
+ if (!path) goto done; // exception was thrown
+
+ dwError = RpcCall_WinutilsChmod(path, (int) jmode);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (path) (*env)->ReleaseStringChars(env, jpath, path);
+
+#endif
+
+}
+
+
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedCopyImpl
+ * Signature: (I;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)V
+ */
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedCopyImpl(JNIEnv* env,
+ jclass clazz, jint joperation, jstring jsourcePath, jstring jdestinationPath, jboolean replaceExisting) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedCopyImpl is not supported on Unix");
+ return NULL;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR sourcePath = NULL, destinationPath = NULL;
+ DWORD dwError;
+
+ sourcePath = (LPCWSTR) (*env)->GetStringChars(env, jsourcePath, NULL);
+ if (!sourcePath) goto done; // exception was thrown
+
+ destinationPath = (LPCWSTR) (*env)->GetStringChars(env, jdestinationPath, NULL);
+ if (!destinationPath) goto done; // exception was thrown
+
+ dwError = RpcCall_WinutilsMoveFile((INT) joperation, sourcePath, destinationPath, (BOOL) replaceExisting);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (sourcePath) (*env)->ReleaseStringChars(env, jsourcePath, sourcePath);
+ if (destinationPath) (*env)->ReleaseStringChars(env, jdestinationPath, destinationPath);
+#endif
+}
+
+/*
+ * Class: Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated
+ * Method: elevatedCreateImpl
+ * Signature: (Ljava/lang/String;J;J;J;J)J
+ */
+JNIEXPORT jlong JNICALL
+Java_org_apache_hadoop_io_nativeio_NativeIO_00024Elevated_elevatedCreateImpl(JNIEnv* env,
+ jclass clazz, jstring jpath, jlong jdesired_access, jlong jshare_mode, jlong jcreation_disposition, jlong jflags) {
+#ifdef UNIX
+ THROW(env, "java/io/IOException",
+ "The function elevatedCreateImpl is not supported on Unix");
+ return INVALID_HANDLE_VALUE;
+#endif
+
+#ifdef WINDOWS
+
+ LPCWSTR path = NULL;
+ DWORD dwError;
+ HANDLE hFile = INVALID_HANDLE_VALUE;
+
+ path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
+ if (!path) goto done; // exception was thrown
+
+ dwError = RpcCall_WinutilsCreateFile(path,
+ (DWORD) jdesired_access, (DWORD) jshare_mode, (DWORD) jcreation_disposition, (DWORD) jflags,
+ &hFile);
+
+ if (dwError != ERROR_SUCCESS) {
+ throw_ioe (env, dwError);
+ }
+
+done:
+ if (path) (*env)->ReleaseStringChars(env, jpath, path);
+ return hFile;
+#endif
+}
/**
* vim: sw=2: ts=2: et:
diff --git hadoop-common-project/hadoop-common/src/main/winutils/chown.c hadoop-common-project/hadoop-common/src/main/winutils/chown.c
index 1be8121..d124f73 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/chown.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/chown.c
@@ -18,93 +18,6 @@
#include "winutils.h"
//----------------------------------------------------------------------------
-// Function: ChangeFileOwnerBySid
-//
-// Description:
-// Change a file or directory ownership by giving new owner and group SIDs
-//
-// Returns:
-// ERROR_SUCCESS: on success
-// Error code: otherwise
-//
-// Notes:
-// This function is long path safe, i.e. the path will be converted to long
-// path format if not already converted. So the caller does not need to do
-// the converstion before calling the method.
-//
-static DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
- __in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid)
-{
- LPWSTR longPathName = NULL;
- INT oldMode = 0;
-
- SECURITY_INFORMATION securityInformation = 0;
-
- DWORD dwRtnCode = ERROR_SUCCESS;
-
- // Convert the path the the long path
- //
- dwRtnCode = ConvertToLongPath(path, &longPathName);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- goto ChangeFileOwnerByNameEnd;
- }
-
- // Get a pointer to the existing owner information and DACL
- //
- dwRtnCode = FindFileOwnerAndPermission(longPathName, FALSE, NULL, NULL, &oldMode);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- goto ChangeFileOwnerByNameEnd;
- }
-
- // We need SeTakeOwnershipPrivilege to set the owner if the caller does not
- // have WRITE_OWNER access to the object; we need SeRestorePrivilege if the
- // SID is not contained in the caller's token, and have the SE_GROUP_OWNER
- // permission enabled.
- //
- if (EnablePrivilege(L"SeTakeOwnershipPrivilege") != ERROR_SUCCESS)
- {
- fwprintf(stdout, L"INFO: The user does not have SeTakeOwnershipPrivilege.\n");
- }
- if (EnablePrivilege(L"SeRestorePrivilege") != ERROR_SUCCESS)
- {
- fwprintf(stdout, L"INFO: The user does not have SeRestorePrivilege.\n");
- }
-
- assert(pNewOwnerSid != NULL || pNewGroupSid != NULL);
-
- // Set the owners of the file.
- //
- if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION;
- if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION;
- dwRtnCode = SetNamedSecurityInfoW(
- longPathName,
- SE_FILE_OBJECT,
- securityInformation,
- pNewOwnerSid,
- pNewGroupSid,
- NULL,
- NULL);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- goto ChangeFileOwnerByNameEnd;
- }
-
- // Set the permission on the file for the new owner.
- //
- dwRtnCode = ChangeFileModeByMask(longPathName, oldMode);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- goto ChangeFileOwnerByNameEnd;
- }
-
-ChangeFileOwnerByNameEnd:
- LocalFree(longPathName);
- return dwRtnCode;
-}
-
-//----------------------------------------------------------------------------
// Function: Chown
//
// Description:
@@ -130,9 +43,6 @@ int Chown(__in int argc, __in_ecount(argc) wchar_t *argv[])
LPWSTR groupName = NULL;
size_t groupNameLen = 0;
- PSID pNewOwnerSid = NULL;
- PSID pNewGroupSid = NULL;
-
DWORD dwRtnCode = 0;
int ret = EXIT_FAILURE;
@@ -210,48 +120,16 @@ int Chown(__in int argc, __in_ecount(argc) wchar_t *argv[])
goto ChownEnd;
}
- if (userName != NULL)
- {
- dwRtnCode = GetSidFromAcctNameW(userName, &pNewOwnerSid);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- ReportErrorCode(L"GetSidFromAcctName", dwRtnCode);
- fwprintf(stderr, L"Invalid user name: %s\n", userName);
- goto ChownEnd;
- }
- }
-
- if (groupName != NULL)
- {
- dwRtnCode = GetSidFromAcctNameW(groupName, &pNewGroupSid);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- ReportErrorCode(L"GetSidFromAcctName", dwRtnCode);
- fwprintf(stderr, L"Invalid group name: %s\n", groupName);
- goto ChownEnd;
- }
- }
-
- if (wcslen(pathName) == 0 || wcsspn(pathName, L"/?|><:*\"") != 0)
- {
- fwprintf(stderr, L"Incorrect file name format: %s\n", pathName);
- goto ChownEnd;
- }
-
- dwRtnCode = ChangeFileOwnerBySid(pathName, pNewOwnerSid, pNewGroupSid);
- if (dwRtnCode != ERROR_SUCCESS)
- {
- ReportErrorCode(L"ChangeFileOwnerBySid", dwRtnCode);
- goto ChownEnd;
- }
+ dwRtnCode = ChownImpl(userName, groupName, pathName);
+ if (dwRtnCode) {
+ goto ChownEnd;
+ }
ret = EXIT_SUCCESS;
ChownEnd:
LocalFree(userName);
LocalFree(groupName);
- LocalFree(pNewOwnerSid);
- LocalFree(pNewGroupSid);
return ret;
}
diff --git hadoop-common-project/hadoop-common/src/main/winutils/client.c hadoop-common-project/hadoop-common/src/main/winutils/client.c
index bc8efcd..bfe48a3 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/client.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/client.c
@@ -44,37 +44,334 @@ VOID ReportClientError(LPWSTR lpszLocation, DWORD dwError) {
if (NULL != debugMsg) LocalFree(debugMsg);
}
+DWORD PrepareRpcBindingHandle(
+ __out RPC_BINDING_HANDLE* pHadoopWinutilsSvcBinding) {
+ DWORD dwError = EXIT_FAILURE;
+ RPC_STATUS status;
+ LPWSTR lpszStringBinding = NULL;
+ ULONG ulCode;
+ RPC_SECURITY_QOS_V3 qos;
+ SID_IDENTIFIER_AUTHORITY authNT = SECURITY_NT_AUTHORITY;
+ BOOL rpcBindingInit = FALSE;
+ PSID pLocalSystemSid = NULL;
+ DWORD cbSystemSidSize = SECURITY_MAX_SID_SIZE;
+
+ pLocalSystemSid = (PSID) LocalAlloc(LPTR, cbSystemSidSize);
+ if (!pLocalSystemSid) {
+ dwError = GetLastError();
+ ReportClientError(L"LocalAlloc", dwError);
+ goto done;
+ }
+
+ if (!CreateWellKnownSid(WinLocalSystemSid, NULL, pLocalSystemSid, &cbSystemSidSize)) {
+ dwError = GetLastError();
+ ReportClientError(L"CreateWellKnownSid", dwError);
+ goto done;
+ }
+
+ ZeroMemory(&qos, sizeof(qos));
+ qos.Version = RPC_C_SECURITY_QOS_VERSION_3;
+ qos.Capabilities = RPC_C_QOS_CAPABILITIES_LOCAL_MA_HINT | RPC_C_QOS_CAPABILITIES_MUTUAL_AUTH;
+ qos.IdentityTracking = RPC_C_QOS_IDENTITY_DYNAMIC;
+ qos.ImpersonationType = RPC_C_IMP_LEVEL_DEFAULT;
+ qos.Sid = pLocalSystemSid;
+
+ status = RpcStringBindingCompose(NULL,
+ SVCBINDING,
+ NULL,
+ SVCNAME,
+ NULL,
+ &lpszStringBinding);
+ if (RPC_S_OK != status) {
+ ReportClientError(L"RpcStringBindingCompose", status);
+ dwError = status;
+ goto done;
+ }
+
+ status = RpcBindingFromStringBinding(lpszStringBinding, pHadoopWinutilsSvcBinding);
+
+ if (RPC_S_OK != status) {
+ ReportClientError(L"RpcBindingFromStringBinding", status);
+ dwError = status;
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ status = RpcBindingSetAuthInfoEx(
+ *pHadoopWinutilsSvcBinding,
+ NULL,
+ RPC_C_AUTHN_LEVEL_PKT_PRIVACY, // AuthnLevel
+ RPC_C_AUTHN_WINNT, // AuthnSvc
+ NULL, // AuthnIdentity (self)
+ RPC_C_AUTHZ_NONE, // AuthzSvc
+ &qos);
+ if (RPC_S_OK != status) {
+ ReportClientError(L"RpcBindingSetAuthInfoEx", status);
+ dwError = status;
+ goto done;
+ }
+
+ dwError = ERROR_SUCCESS;
+
+done:
+
+ if (dwError && rpcBindingInit) RpcBindingFree(pHadoopWinutilsSvcBinding);
+
+ if (pLocalSystemSid) LocalFree(pLocalSystemSid);
+
+ if (NULL != lpszStringBinding) {
+ status = RpcStringFree(&lpszStringBinding);
+ if (RPC_S_OK != status) {
+ ReportClientError(L"RpcStringFree", status);
+ }
+ }
+
+ return dwError;
+}
+
+
+DWORD RpcCall_WinutilsMkDir(
+ __in LPCWSTR filePath) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ MKDIR_REQUEST request;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.filePath = filePath;
+
+ RpcTryExcept {
+ dwError = WinutilsMkDir(hHadoopWinutilsSvcBinding, &request);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ LogDebugMessage(L"RpcCall_WinutilsMkDir: %s :%d\n", filePath, dwError);
+
+ return dwError;
+}
+
+
+
+DWORD RpcCall_WinutilsChown(
+ __in LPCWSTR filePath,
+ __in_opt LPCWSTR ownerName,
+ __in_opt LPCWSTR groupName) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ CHOWN_REQUEST request;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.filePath = filePath;
+ request.ownerName = ownerName;
+ request.groupName = groupName;
+
+ RpcTryExcept {
+ dwError = WinutilsChown(hHadoopWinutilsSvcBinding, &request);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ LogDebugMessage(L"RpcCall_WinutilsChown: %s %s %s :%d\n",
+ ownerName, groupName, filePath, dwError);
+
+ return dwError;
+}
+
+
+DWORD RpcCall_WinutilsChmod(
+ __in LPCWSTR filePath,
+ __in int mode) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ CHMOD_REQUEST request;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.filePath = filePath;
+ request.mode = mode;
+
+ RpcTryExcept {
+ dwError = WinutilsChown(hHadoopWinutilsSvcBinding, &request);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ LogDebugMessage(L"RpcCall_WinutilsChmod: %s %o :%d\n",
+ filePath, mode, dwError);
+
+ return dwError;
+}
+
+
+
+DWORD RpcCall_WinutilsMoveFile(
+ __in int operation,
+ __in LPCWSTR sourcePath,
+ __in LPCWSTR destinationPath,
+ __in BOOL replaceExisting) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ MOVEFILE_REQUEST request;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.operation = operation;
+ request.sourcePath = sourcePath;
+ request.destinationPath = destinationPath;
+ request.replaceExisting = replaceExisting;
+
+ RpcTryExcept {
+ dwError = WinutilsMoveFile(hHadoopWinutilsSvcBinding, &request);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ LogDebugMessage(L"RpcCall_WinutilsMoveFile: %s %s %d :%d\n",
+ sourcePath, destinationPath, replaceExisting, dwError);
+
+ return dwError;
+}
+
+DWORD RpcCall_WinutilsCreateFile(
+ __in LPCWSTR path,
+ __in DWORD desiredAccess,
+ __in DWORD shareMode,
+ __in DWORD creationDisposition,
+ __in DWORD flags,
+ __out HANDLE* hFile) {
+
+ DWORD dwError = EXIT_FAILURE;
+ ULONG ulCode;
+ DWORD dwSelfPid = GetCurrentProcessId();
+ CREATEFILE_REQUEST request;
+ CREATEFILE_RESPONSE *response = NULL;
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
+ goto done;
+ }
+ rpcBindingInit = TRUE;
+
+ ZeroMemory(&request, sizeof(request));
+ request.path = path;
+ request.desiredAccess = desiredAccess;
+ request.shareMode = shareMode;
+ request.creationDisposition = creationDisposition;
+ request.flags = flags;
+
+ RpcTryExcept {
+ dwError = WinutilsCreateFile(hHadoopWinutilsSvcBinding, dwSelfPid, &request, &response);
+ }
+ RpcExcept(1) {
+ ulCode = RpcExceptionCode();
+ ReportClientError(L"RpcExcept", ulCode);
+ dwError = (DWORD) ulCode;
+ }
+ RpcEndExcept;
+
+ if (ERROR_SUCCESS == dwError) {
+ *hFile = response->hFile;
+ }
+
+done:
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
+ if(NULL != response) MIDL_user_free(response);
+
+ LogDebugMessage(L"RpcCall_WinutilsCreateFile: %s %d, %d, %d, %d :%d\n",
+ path, desiredAccess, shareMode, creationDisposition, flags, dwError);
+
+ return dwError;
+}
+
DWORD RpcCall_TaskCreateAsUser(
LPCWSTR cwd, LPCWSTR jobName,
LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine,
HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr)
{
DWORD dwError = EXIT_FAILURE;
- RPC_STATUS status;
- LPWSTR lpszStringBinding = NULL;
ULONG ulCode;
DWORD dwSelfPid = GetCurrentProcessId();
CREATE_PROCESS_REQUEST request;
CREATE_PROCESS_RESPONSE *response = NULL;
- RPC_SECURITY_QOS_V3 qos;
- PSID pLocalSystemSid = NULL;
- SID_IDENTIFIER_AUTHORITY authNT = SECURITY_NT_AUTHORITY;
-
- if (!AllocateAndInitializeSid(&authNT, 1,
- SECURITY_LOCAL_SYSTEM_RID,
- 0, 0, 0, 0, 0, 0, 0,
- &pLocalSystemSid)) {
- dwError = GetLastError();
- ReportClientError(L"AllocateAndInitializeSid", dwError);
+ RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
+ BOOL rpcBindingInit = FALSE;
+
+ dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
+ if (dwError) {
+ ReportClientError(L"PrepareRpcBindingHandle", dwError);
goto done;
}
-
- ZeroMemory(&qos, sizeof(qos));
- qos.Version = RPC_C_SECURITY_QOS_VERSION_3;
- qos.Capabilities = RPC_C_QOS_CAPABILITIES_LOCAL_MA_HINT | RPC_C_QOS_CAPABILITIES_MUTUAL_AUTH;
- qos.IdentityTracking = RPC_C_QOS_IDENTITY_DYNAMIC;
- qos.ImpersonationType = RPC_C_IMP_LEVEL_DEFAULT;
- qos.Sid = pLocalSystemSid;
+ rpcBindingInit = TRUE;
ZeroMemory(&request, sizeof(request));
request.cwd = cwd;
@@ -83,42 +380,8 @@ DWORD RpcCall_TaskCreateAsUser(
request.pidFile = pidFile;
request.cmdLine = cmdLine;
- status = RpcStringBindingCompose(NULL,
- SVCBINDING,
- NULL,
- SVCNAME,
- NULL,
- &lpszStringBinding);
- if (RPC_S_OK != status) {
- ReportClientError(L"RpcStringBindingCompose", status);
- dwError = status;
- goto done;
- }
-
- status = RpcBindingFromStringBinding(lpszStringBinding, &hHadoopWinutilsSvcBinding);
-
- if (RPC_S_OK != status) {
- ReportClientError(L"RpcBindingFromStringBinding", status);
- dwError = status;
- goto done;
- }
-
- status = RpcBindingSetAuthInfoEx(
- hHadoopWinutilsSvcBinding,
- NULL,
- RPC_C_AUTHN_LEVEL_PKT_PRIVACY, // AuthnLevel
- RPC_C_AUTHN_WINNT, // AuthnSvc
- NULL, // AuthnIdentity (self)
- RPC_C_AUTHZ_NONE, // AuthzSvc
- &qos);
- if (RPC_S_OK != status) {
- ReportClientError(L"RpcBindingSetAuthInfoEx", status);
- dwError = status;
- goto done;
- }
-
RpcTryExcept {
- dwError = WinutilsCreateProcessAsUser(dwSelfPid, &request, &response);
+ dwError = WinutilsCreateProcessAsUser(hHadoopWinutilsSvcBinding, dwSelfPid, &request, &response);
}
RpcExcept(1) {
ulCode = RpcExceptionCode();
@@ -135,27 +398,13 @@ DWORD RpcCall_TaskCreateAsUser(
*phStdErr = response->hStdErr;
}
- // From here on forward we do no change dwError even on RPC cleanup errors
- status = RpcBindingFree(&hHadoopWinutilsSvcBinding);
- if (RPC_S_OK != status) {
- ReportClientError(L"RpcBindingFree", status);
- goto done;
- }
-
done:
- if (pLocalSystemSid) FreeSid(pLocalSystemSid);
-
+ if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
+
if (NULL != response) {
MIDL_user_free(response);
}
- if (NULL != lpszStringBinding) {
- status = RpcStringFree(&lpszStringBinding);
- if (RPC_S_OK != status) {
- ReportClientError(L"RpcStringFree", status);
- }
- }
-
return dwError;
}
diff --git hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl
index 2285178..ec7128f 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl
+++ hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl
@@ -1,35 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
import "oaidl.idl";
import "ocidl.idl";
[
- uuid(0492311C-1718-4F53-A6EB-86AD7039988D),
- version(1.0),
- pointer_default(unique),
- implicit_handle(handle_t hHadoopWinutilsSvcBinding),
- endpoint("ncalrpc:[hadoopwinutilsvc]"),
+ uuid(0492311C-1718-4F53-A6EB-86AD7039988D),
+ version(1.0),
+ pointer_default(unique),
+ //implicit_handle(handle_t hHadoopWinutilsSvcBinding),
+ endpoint("ncalrpc:[hadoopwinutilsvc]"),
+#ifndef __midl
+ explicit_handle
+#endif
]
interface HadoopWinutilSvc
{
- typedef struct {
- [string] const wchar_t* cwd;
- [string] const wchar_t* jobName;
- [string] const wchar_t* user;
- [string] const wchar_t* pidFile;
- [string] const wchar_t* cmdLine;
- } CREATE_PROCESS_REQUEST;
-
- typedef struct {
- LONG_PTR hProcess;
- LONG_PTR hThread;
- LONG_PTR hStdIn;
- LONG_PTR hStdOut;
- LONG_PTR hStdErr;
- } CREATE_PROCESS_RESPONSE;
-
-
- error_status_t WinutilsCreateProcessAsUser(
- [in] int nmPid,
- [in] CREATE_PROCESS_REQUEST *request,
- [out] CREATE_PROCESS_RESPONSE **response);
-
-}
\ No newline at end of file
+ typedef struct {
+ [string] const wchar_t* cwd;
+ [string] const wchar_t* jobName;
+ [string] const wchar_t* user;
+ [string] const wchar_t* pidFile;
+ [string] const wchar_t* cmdLine;
+ } CREATE_PROCESS_REQUEST;
+
+ typedef struct {
+ LONG_PTR hProcess;
+ LONG_PTR hThread;
+ LONG_PTR hStdIn;
+ LONG_PTR hStdOut;
+ LONG_PTR hStdErr;
+ } CREATE_PROCESS_RESPONSE;
+
+ typedef struct {
+ [string] const wchar_t* filePath;
+ [string] const wchar_t* ownerName;
+ [string] const wchar_t* groupName;
+ } CHOWN_REQUEST;
+
+ typedef struct {
+ [string] const wchar_t* filePath;
+ int mode;
+ } CHMOD_REQUEST;
+
+ typedef struct {
+ [string] const wchar_t* filePath;
+ } MKDIR_REQUEST;
+
+ typedef enum { MOVE_FILE = 1, COPY_FILE = 2} MOVE_COPY_OPERATION;
+
+ typedef struct {
+ MOVE_COPY_OPERATION operation;
+ [string] const wchar_t* sourcePath;
+ [string] const wchar_t* destinationPath;
+ boolean replaceExisting;
+ } MOVEFILE_REQUEST;
+
+ typedef struct {
+ [string] const wchar_t* path;
+ int desiredAccess;
+ int shareMode;
+ int creationDisposition;
+ int flags;
+ } CREATEFILE_REQUEST;
+
+ typedef struct {
+ LONG_PTR hFile;
+ } CREATEFILE_RESPONSE;
+
+ error_status_t WinutilsMkDir(
+ [in] MKDIR_REQUEST *request);
+
+ error_status_t WinutilsMoveFile(
+ [in] MOVEFILE_REQUEST *request);
+
+ error_status_t WinutilsChown(
+ [in] CHOWN_REQUEST *request);
+
+ error_status_t WinutilsChmod(
+ [in] CHMOD_REQUEST *request);
+
+ error_status_t WinutilsCreateFile(
+ [in] int nmPid,
+ [in] CREATEFILE_REQUEST *request,
+ [out] CREATEFILE_RESPONSE **response);
+
+ error_status_t WinutilsCreateProcessAsUser(
+ [in] int nmPid,
+ [in] CREATE_PROCESS_REQUEST *request,
+ [out] CREATE_PROCESS_RESPONSE **response);
+
+}
diff --git hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
index 51835b6..7be86c3 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
+++ hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
@@ -34,6 +34,7 @@
extern "C" {
#endif
+
enum EXIT_CODE
{
/* Common success exit code shared among all utilities */
@@ -42,6 +43,12 @@ enum EXIT_CODE
FAILURE = EXIT_FAILURE,
/* Failure code indicates the user does not privilege to create symlinks */
SYMLINK_NO_PRIVILEGE = 2,
+
+ ERROR_TASK_NOT_ALIVE = 1,
+
+ // This exit code for killed processes is compatible with Unix, where a killed
+ // process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137.
+ KILLED_PROCESS_EXIT_CODE = 137,
};
@@ -185,6 +192,15 @@ DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi);
DWORD RunService(__in int argc, __in_ecount(argc) wchar_t *argv[]);
void ServiceUsage();
+
+DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
+ __in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid);
+
+DWORD ChownImpl(
+ __in_opt LPCWSTR userName,
+ __in_opt LPCWSTR groupName,
+ __in LPCWSTR pathName);
+
LPCWSTR GetSystemTimeString();
VOID LogDebugMessage(LPCWSTR format, ...);
@@ -217,14 +233,30 @@ DWORD BuildServiceSecurityDescriptor(
extern const WCHAR* wsceConfigRelativePath;
+extern LPCWSTR NM_WSCE_ALLOWED;
+
+
#define SVCNAME TEXT("hadoopwinutilsvc")
#define SVCBINDING TEXT("ncalrpc")
-int RpcCall_TaskCreateAsUser(
+DWORD RpcCall_TaskCreateAsUser(
LPCWSTR cwd, LPCWSTR jobName,
LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine,
HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr);
+DWORD RpcCall_WinutilsCreateFile(
+ __in LPCWSTR path,
+ __in DWORD desiredAccess,
+ __in DWORD shareMode,
+ __in DWORD creationDisposition,
+ __in DWORD flags,
+ __out HANDLE* hFile);
+
+DWORD RpcCall_WinutilsMoveFile(
+ __in LPCWSTR sourcePath,
+ __in LPCWSTR destinationPath,
+ __in BOOL replaceExisting);
+
#ifdef __cplusplus
}
#endif
diff --git hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
index 19aae4c..78a3de3 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
@@ -252,10 +252,10 @@ ConvertToLongPathExit:
// Function: IsDirFileInfo
//
// Description:
-// Test if the given file information is a directory
+// Test if the given file information is a directory
//
// Returns:
-// TRUE if it is a directory
+// TRUE if it is a directory
// FALSE otherwise
//
// Notes:
@@ -272,10 +272,10 @@ BOOL IsDirFileInfo(const BY_HANDLE_FILE_INFORMATION *fileInformation)
// Function: CheckFileAttributes
//
// Description:
-// Check if the given file has all the given attribute(s)
+// Check if the given file has all the given attribute(s)
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -296,10 +296,10 @@ static DWORD FileAttributesCheck(
// Function: IsDirectory
//
// Description:
-// Check if the given file is a directory
+// Check if the given file is a directory
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -313,10 +313,10 @@ DWORD DirectoryCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: IsReparsePoint
//
// Description:
-// Check if the given file is a reparse point
+// Check if the given file is a reparse point
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -330,10 +330,10 @@ static DWORD ReparsePointCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: CheckReparseTag
//
// Description:
-// Check if the given file is a reparse point of the given tag.
+// Check if the given file is a reparse point of the given tag.
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -371,10 +371,10 @@ static DWORD ReparseTagCheck(__in LPCWSTR path, __in DWORD tag, __out PBOOL res)
// Function: IsSymbolicLink
//
// Description:
-// Check if the given file is a symbolic link.
+// Check if the given file is a symbolic link.
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -388,10 +388,10 @@ DWORD SymbolicLinkCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: IsJunctionPoint
//
// Description:
-// Check if the given file is a junction point.
+// Check if the given file is a junction point.
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@@ -405,14 +405,14 @@ DWORD JunctionPointCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: GetSidFromAcctNameW
//
// Description:
-// To retrieve the SID for a user account
+// To retrieve the SID for a user account
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
// Other error code: otherwise
//
// Notes:
-// Caller needs to destroy the memory of Sid by calling LocalFree()
+// Caller needs to destroy the memory of Sid by calling LocalFree()
//
DWORD GetSidFromAcctNameW(__in PCWSTR acctName, __out PSID *ppSid)
{
@@ -494,10 +494,10 @@ DWORD GetSidFromAcctNameW(__in PCWSTR acctName, __out PSID *ppSid)
// Function: GetUnixAccessMask
//
// Description:
-// Compute the 3 bit Unix mask for the owner, group, or, others
+// Compute the 3 bit Unix mask for the owner, group, or, others
//
// Returns:
-// The 3 bit Unix mask in INT
+// The 3 bit Unix mask in INT
//
// Notes:
//
@@ -521,10 +521,10 @@ static INT GetUnixAccessMask(ACCESS_MASK Mask)
// Function: GetAccess
//
// Description:
-// Get Windows acces mask by AuthZ methods
+// Get Windows acces mask by AuthZ methods
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
//
// Notes:
//
@@ -569,10 +569,10 @@ static DWORD GetAccess(AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzClient,
// Function: GetEffectiveRightsForSid
//
// Description:
-// Get Windows acces mask by AuthZ methods
+// Get Windows acces mask by AuthZ methods
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
//
// Notes:
// We run into problems for local user accounts when using the method
@@ -729,11 +729,11 @@ CheckAccessEnd:
// Function: FindFileOwnerAndPermissionByHandle
//
// Description:
-// Find the owner, primary group and permissions of a file object given the
+// Find the owner, primary group and permissions of a file object given the
// the file object handle. The function will always follow symbolic links.
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
// Error code otherwise
//
// Notes:
@@ -793,10 +793,10 @@ FindFileOwnerAndPermissionByHandleEnd:
// Function: FindFileOwnerAndPermission
//
// Description:
-// Find the owner, primary group and permissions of a file object
+// Find the owner, primary group and permissions of a file object
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
// Error code otherwise
//
// Notes:
@@ -1222,14 +1222,14 @@ static DWORD GetWindowsDACLs(__in INT unixMask,
if (winUserAccessDenyMask &&
!AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winUserAccessDenyMask, pOwnerSid))
{
ret = GetLastError();
goto GetWindowsDACLsEnd;
}
if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winUserAccessAllowMask, pOwnerSid))
{
ret = GetLastError();
@@ -1237,21 +1237,21 @@ static DWORD GetWindowsDACLs(__in INT unixMask,
}
if (winGroupAccessDenyMask &&
!AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winGroupAccessDenyMask, pGroupSid))
{
ret = GetLastError();
goto GetWindowsDACLsEnd;
}
if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winGroupAccessAllowMask, pGroupSid))
{
ret = GetLastError();
goto GetWindowsDACLsEnd;
}
if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
- NO_PROPAGATE_INHERIT_ACE,
+ CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winOtherAccessAllowMask, pEveryoneSid))
{
ret = GetLastError();
@@ -1455,14 +1455,14 @@ ChangeFileModeByMaskEnd:
// Function: GetAccntNameFromSid
//
// Description:
-// To retrieve an account name given the SID
+// To retrieve an account name given the SID
//
// Returns:
-// ERROR_SUCCESS: on success
+// ERROR_SUCCESS: on success
// Other error code: otherwise
//
// Notes:
-// Caller needs to destroy the memory of account name by calling LocalFree()
+// Caller needs to destroy the memory of account name by calling LocalFree()
//
DWORD GetAccntNameFromSid(__in PSID pSid, __out PWSTR *ppAcctName)
{
@@ -1551,10 +1551,10 @@ GetAccntNameFromSidEnd:
// Function: GetLocalGroupsForUser
//
// Description:
-// Get an array of groups for the given user.
+// Get an array of groups for the given user.
//
// Returns:
-// ERROR_SUCCESS on success
+// ERROR_SUCCESS on success
// Other error code on failure
//
// Notes:
@@ -1646,11 +1646,12 @@ GetLocalGroupsForUserEnd:
return ret;
}
+
//----------------------------------------------------------------------------
// Function: EnablePrivilege
//
// Description:
-// Check if the process has the given privilege. If yes, enable the privilege
+// Check if the process has the given privilege. If yes, enable the privilege
// to the process's access token.
//
// Returns:
@@ -2068,6 +2069,148 @@ done:
}
+//----------------------------------------------------------------------------
+// Function: ChangeFileOwnerBySid
+//
+// Description:
+// Change a file or directory ownership by giving new owner and group SIDs
+//
+// Returns:
+// ERROR_SUCCESS: on success
+// Error code: otherwise
+//
+// Notes:
+// This function is long path safe, i.e. the path will be converted to long
+// path format if not already converted. So the caller does not need to do
+// the converstion before calling the method.
+//
+DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
+ __in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid)
+{
+ LPWSTR longPathName = NULL;
+ INT oldMode = 0;
+
+ SECURITY_INFORMATION securityInformation = 0;
+
+ DWORD dwRtnCode = ERROR_SUCCESS;
+
+ // Convert the path the the long path
+ //
+ dwRtnCode = ConvertToLongPath(path, &longPathName);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ goto ChangeFileOwnerByNameEnd;
+ }
+
+ // Get a pointer to the existing owner information and DACL
+ //
+ dwRtnCode = FindFileOwnerAndPermission(longPathName, FALSE, NULL, NULL, &oldMode);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ goto ChangeFileOwnerByNameEnd;
+ }
+
+ // We need SeTakeOwnershipPrivilege to set the owner if the caller does not
+ // have WRITE_OWNER access to the object; we need SeRestorePrivilege if the
+ // SID is not contained in the caller's token, and have the SE_GROUP_OWNER
+ // permission enabled.
+ //
+ if (EnablePrivilege(L"SeTakeOwnershipPrivilege") != ERROR_SUCCESS)
+ {
+ fwprintf(stdout, L"INFO: The user does not have SeTakeOwnershipPrivilege.\n");
+ }
+ if (EnablePrivilege(L"SeRestorePrivilege") != ERROR_SUCCESS)
+ {
+ fwprintf(stdout, L"INFO: The user does not have SeRestorePrivilege.\n");
+ }
+
+ assert(pNewOwnerSid != NULL || pNewGroupSid != NULL);
+
+ // Set the owners of the file.
+ //
+ if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION;
+ if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION;
+ dwRtnCode = SetNamedSecurityInfoW(
+ longPathName,
+ SE_FILE_OBJECT,
+ securityInformation,
+ pNewOwnerSid,
+ pNewGroupSid,
+ NULL,
+ NULL);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ goto ChangeFileOwnerByNameEnd;
+ }
+
+ // Set the permission on the file for the new owner.
+ //
+ dwRtnCode = ChangeFileModeByMask(longPathName, oldMode);
+ if (dwRtnCode != ERROR_SUCCESS)
+ {
+ goto ChangeFileOwnerByNameEnd;
+ }
+
+ChangeFileOwnerByNameEnd:
+ LocalFree(longPathName);
+ return dwRtnCode;
+}
+
+
+
+DWORD ChownImpl(
+ __in_opt LPCWSTR userName,
+ __in_opt LPCWSTR groupName,
+ __in LPCWSTR pathName) {
+
+ DWORD dwError;
+
+ PSID pNewOwnerSid = NULL;
+ PSID pNewGroupSid = NULL;
+
+ if (userName != NULL)
+ {
+ dwError = GetSidFromAcctNameW(userName, &pNewOwnerSid);
+ if (dwError != ERROR_SUCCESS)
+ {
+ ReportErrorCode(L"GetSidFromAcctName", dwError);
+ fwprintf(stderr, L"Invalid user name: %s\n", userName);
+ goto done;
+ }
+ }
+
+ if (groupName != NULL)
+ {
+ dwError = GetSidFromAcctNameW(groupName, &pNewGroupSid);
+ if (dwError != ERROR_SUCCESS)
+ {
+ ReportErrorCode(L"GetSidFromAcctName", dwError);
+ fwprintf(stderr, L"Invalid group name: %s\n", groupName);
+ goto done;
+ }
+ }
+
+ if (wcslen(pathName) == 0 || wcsspn(pathName, L"/?|><:*\"") != 0)
+ {
+ fwprintf(stderr, L"Incorrect file name format: %s\n", pathName);
+ goto done;
+ }
+
+ dwError = ChangeFileOwnerBySid(pathName, pNewOwnerSid, pNewGroupSid);
+ if (dwError != ERROR_SUCCESS)
+ {
+ ReportErrorCode(L"ChangeFileOwnerBySid", dwError);
+ goto done;
+ }
+done:
+ LocalFree(pNewOwnerSid);
+ LocalFree(pNewGroupSid);
+
+ return dwError;
+}
+
+
+
LPCWSTR GetSystemTimeString() {
__declspec(thread) static WCHAR buffer[1024];
DWORD dwError;
@@ -2362,7 +2505,7 @@ DWORD BuildServiceSecurityDescriptor(
group.TrusteeType = TRUSTEE_IS_UNKNOWN;
group.ptstrName = (LPCWSTR) pTokenGroup->PrimaryGroup;
- eas = (EXPLICIT_ACCESS*) alloca(sizeof(EXPLICIT_ACCESS) * (grantSidCount + denySidCount));
+ eas = (EXPLICIT_ACCESS*) LocalAlloc(LPTR, sizeof(EXPLICIT_ACCESS) * (grantSidCount + denySidCount));
// Build the granted list
for (crt = 0; crt < grantSidCount; ++crt) {
@@ -2372,6 +2515,8 @@ DWORD BuildServiceSecurityDescriptor(
eas[crt].Trustee.TrusteeForm = TRUSTEE_IS_SID;
eas[crt].Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN;
eas[crt].Trustee.ptstrName = (LPCWSTR) pGrantSids[crt];
+ eas[crt].Trustee.pMultipleTrustee = NULL;
+ eas[crt].Trustee.MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE;
}
// Build the deny list
@@ -2382,6 +2527,8 @@ DWORD BuildServiceSecurityDescriptor(
eas[crt].Trustee.TrusteeForm = TRUSTEE_IS_SID;
eas[crt].Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN;
eas[crt].Trustee.ptstrName = (LPCWSTR) pDenySids[crt - grantSidCount];
+ eas[crt].Trustee.pMultipleTrustee = NULL;
+ eas[crt].Trustee.MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE;
}
dwError = BuildSecurityDescriptor(
@@ -2411,6 +2558,7 @@ DWORD BuildServiceSecurityDescriptor(
}
done:
+ if (eas) LocalFree(eas);
if (pTokenUser) LocalFree(pTokenUser);
if (INVALID_HANDLE_VALUE != hToken) CloseHandle(hToken);
if (lpszSD) LocalFree(lpszSD);
diff --git hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
index cbc4ae9..0b05be0 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
+++ hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
@@ -80,7 +80,8 @@
Level3
- MaxSpeed
+
+ Disabled
true
true
WIN32;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)
diff --git hadoop-common-project/hadoop-common/src/main/winutils/service.c hadoop-common-project/hadoop-common/src/main/winutils/service.c
index 70c2a1d..37b8628 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/service.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/service.c
@@ -29,9 +29,7 @@
#pragma comment(lib, "advapi32.lib")
#pragma comment(lib, "authz.lib")
-
-#define NM_WSCE_ALLOWED L"yarn.nodemanager.windows-secure-container-executor.allowed"
-#define NM_WSCE_DENIED L"yarn.nodemanager.windows-secure-container-executor.denied"
+LPCWSTR NM_WSCE_ALLOWED = L"yarn.nodemanager.windows-secure-container-executor.allowed";
#define SERVICE_ACCESS_MASK 0x00000001
@@ -637,6 +635,7 @@ VOID ReportSvcStatus( DWORD dwCurrentState,
// The actual S4U work occurs in the spawned process, run and monitored by the NM
//
error_status_t WinutilsCreateProcessAsUser(
+ /* [in] */ handle_t IDL_handle,
/* [in] */ int nmPid,
/* [in] */ CREATE_PROCESS_REQUEST *request,
/* [out] */ CREATE_PROCESS_RESPONSE **response) {
@@ -866,6 +865,156 @@ done:
return dwError;
}
+error_status_t WinutilsCreateFile(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ int nm_pid,
+ /* [in] */ CREATEFILE_REQUEST *request,
+ /* [out] */ CREATEFILE_RESPONSE **response) {
+
+ DWORD dwError = ERROR_SUCCESS;
+
+ HANDLE hNmProcess = INVALID_HANDLE_VALUE,
+ hFile = INVALID_HANDLE_VALUE,
+ hDuplicateFile = INVALID_HANDLE_VALUE,
+ hSelfProcess = GetCurrentProcess();
+
+ SECURITY_ATTRIBUTES saFile;
+
+ ZeroMemory( &saFile, sizeof(saFile));
+
+ saFile.nLength = sizeof(SECURITY_ATTRIBUTES);
+ saFile.bInheritHandle = TRUE;
+ saFile.lpSecurityDescriptor = NULL;
+
+ hFile = CreateFile(
+ request->path,
+ request->desiredAccess,
+ request->shareMode,
+ &saFile,
+ request->creationDisposition,
+ request->flags,
+ NULL); // hTemplate
+ if (INVALID_HANDLE_VALUE == hFile) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ hNmProcess = OpenProcess(PROCESS_DUP_HANDLE, FALSE, nm_pid);
+ if (NULL == hNmProcess) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ if (!DuplicateHandle(hSelfProcess, hFile,
+ hNmProcess, &hDuplicateFile,
+ 0, FALSE, DUPLICATE_SAME_ACCESS)) {
+ dwError = GetLastError();
+ goto done;
+ }
+
+ *response = (CREATEFILE_RESPONSE*) MIDL_user_allocate(sizeof(CREATEFILE_RESPONSE));
+ if (NULL == *response) {
+ dwError = ERROR_OUTOFMEMORY;
+ goto done;
+ }
+
+ (*response)->hFile = hDuplicateFile;
+ hDuplicateFile = INVALID_HANDLE_VALUE;
+
+done:
+
+ if (INVALID_HANDLE_VALUE != hFile) CloseHandle(hFile);
+ if (INVALID_HANDLE_VALUE != hDuplicateFile) {
+ DuplicateHandle(hNmProcess, hDuplicateFile, NULL, NULL, 0, FALSE, DUPLICATE_CLOSE_SOURCE);
+ }
+ if (INVALID_HANDLE_VALUE != hNmProcess) CloseHandle(hNmProcess);
+
+ LogDebugMessage(L"WinutilsCreateFile: %s %d, %d, %d, %d: %d",
+ request->path,
+ request->desiredAccess,
+ request->shareMode,
+ request->creationDisposition,
+ request->flags,
+ dwError);
+
+ return dwError;
+}
+
+error_status_t WinutilsMkDir(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ MKDIR_REQUEST *request) {
+ DWORD dwError = ERROR_SUCCESS;
+ if (!CreateDirectory(request->filePath, NULL)) {
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"CreateDirectory");
+ }
+ LogDebugMessage(L"WinutilsMkDir: %s :%d\n", request->filePath, dwError);
+ return dwError;
+}
+
+error_status_t WinutilsChown(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ CHOWN_REQUEST *request) {
+ DWORD dwError = ERROR_SUCCESS;
+ dwError = ChownImpl(request->ownerName, request->groupName, request->filePath);
+ if (dwError) {
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"ChownImpl");
+ }
+ LogDebugMessage(L"WinutilsChown: %s %s %s :%d\n",
+ request->ownerName, request->groupName, request->filePath, dwError);
+ return dwError;
+}
+
+error_status_t WinutilsChmod(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ CHMOD_REQUEST *request) {
+ DWORD dwError = ERROR_SUCCESS;
+ dwError = ChangeFileModeByMask(request->filePath, request->mode);
+ if (dwError) {
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"ChangeFileModeByMask");
+ }
+ LogDebugMessage(L"WinutilsChmod: %s %o :%d\n",
+ request->filePath, request->mode, dwError);
+ return dwError;
+}
+
+error_status_t WinutilsMoveFile(
+ /* [in] */ handle_t IDL_handle,
+ /* [in] */ MOVEFILE_REQUEST *request) {
+ DWORD dwError = ERROR_SUCCESS;
+ DWORD flags = 0;
+
+ switch (request->operation) {
+ case MOVE_FILE:
+ flags |= MOVEFILE_COPY_ALLOWED;
+ if (request->replaceExisting) flags |= MOVEFILE_REPLACE_EXISTING;
+ if (!MoveFileEx(request->sourcePath, request->destinationPath, flags)) {
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"MoveFileEx");
+ }
+ break;
+ case COPY_FILE:
+ if (!request->replaceExisting) flags |= COPY_FILE_FAIL_IF_EXISTS;
+ if (!CopyFileEx(request->sourcePath, request->destinationPath,
+ NULL, // lpProgressRoutine
+ NULL, // lpData
+ NULL, // pbCancel
+ flags)) {
+ dwError = GetLastError();
+ ReportSvcCheckError(EVENTLOG_ERROR_TYPE, SERVICE_CATEGORY,
+ dwError, L"CopyFileEx");
+ }
+ }
+ LogDebugMessage(L"WinutilsMoveFile: %d: %s %s :%d\n",
+ request->operation, request->sourcePath, request->destinationPath, dwError);
+ return dwError;
+}
+
+
//----------------------------------------------------------------------------
// Function: ServiceUsage
//
diff --git hadoop-common-project/hadoop-common/src/main/winutils/task.c hadoop-common-project/hadoop-common/src/main/winutils/task.c
index 67e82c3..67678f2 100644
--- hadoop-common-project/hadoop-common/src/main/winutils/task.c
+++ hadoop-common-project/hadoop-common/src/main/winutils/task.c
@@ -24,18 +24,12 @@
#define PSAPI_VERSION 1
#pragma comment(lib, "psapi.lib")
-
#define NM_WSCE_IMPERSONATE_ALLOWED L"yarn.nodemanager.windows-secure-container-executor.impersonate.allowed"
#define NM_WSCE_IMPERSONATE_DENIED L"yarn.nodemanager.windows-secure-container-executor.impersonate.denied"
// The S4U impersonation access check mask. Arbitrary value (we use 1 for the service access check)
#define SERVICE_IMPERSONATE_MASK 0x00000002
-#define ERROR_TASK_NOT_ALIVE 1
-
-// This exit code for killed processes is compatible with Unix, where a killed
-// process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137.
-#define KILLED_PROCESS_EXIT_CODE 137
// Name for tracking this logon process when registering with LSA
static const char *LOGON_PROCESS_NAME="Hadoop Container Executor";
@@ -230,6 +224,112 @@ done:
}
//----------------------------------------------------------------------------
+// Function: BuildJobObjectSecurityDescriptor
+//
+// Description:
+// Builds the security descriptor for NT job object that contains the task
+// Both the nodemanager and the container user require access to the job object
+// The ACEs grant full controll to NM, container job and LocalSystem (the WSCE winutils service)
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+//
+DWORD BuildJobObjectSecurityDescriptor(
+ __in LPCWSTR user,
+ __out PSECURITY_DESCRIPTOR* pSD) {
+
+ DWORD dwError;
+
+ DWORD cbSid = SECURITY_MAX_SID_SIZE;
+ PSID pSidNodeManager = NULL;
+ PSID pSidLocalSystem = NULL;
+ PSID pSidUser = NULL;
+ PSID* allowedSids = NULL;
+ int countSids = 0;
+ int countTokens = 0;
+ int len = 0;
+ LPCWSTR value = NULL;
+ WCHAR** tokens = NULL;
+ int crt = 0;
+
+ dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_ALLOWED, &len, &value);
+ if (ERROR_SUCCESS != dwError) {
+ ReportErrorCode(L"GetConfigValue", dwError);
+ goto done;
+ }
+
+ if (0 == len) {
+ dwError = ERROR_BAD_CONFIGURATION;
+ ReportErrorCode(L"GetConfigValue", dwError);
+ goto done;
+ }
+
+ dwError = SplitStringIgnoreSpaceW(len, value, L',', &countTokens, &tokens);
+ if (ERROR_SUCCESS != dwError) {
+ ReportErrorCode(L"SplitStringIgnoreSpaceW", dwError);
+ goto done;
+ }
+
+ // allocate for all the configure granted users (usually NM service account)
+ // +1 for the container user
+ // +1 for LocalSystem
+ //
+ allowedSids = (PSID*) LocalAlloc(LPTR, sizeof(PSID) * (countTokens + 2));
+ if (NULL == allowedSids) {
+ dwError = ERROR_OUTOFMEMORY;
+ ReportErrorCode(L"LocalAlloc:pSidLocalSystem", dwError);
+ goto done;
+ }
+
+ for (crt = 0; crt < countTokens; ++crt) {
+ dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]);
+ if (ERROR_SUCCESS != dwError) {
+ ReportErrorCode(L"GetSidFromAcctNameW", dwError);
+ goto done;
+ }
+ }
+
+ dwError = GetSidFromAcctNameW(user, &allowedSids[crt]);
+ if (ERROR_SUCCESS != dwError) {
+ ReportErrorCode(L"GetSidFromAcctNameW:user", dwError);
+ goto done;
+ }
+
+ ++crt;
+
+ allowedSids[crt] = (PSID) LocalAlloc(LPTR, SECURITY_MAX_SID_SIZE);
+ if (NULL == allowedSids[crt]) {
+ dwError = ERROR_OUTOFMEMORY;
+ ReportErrorCode(L"LocalAlloc:pSidLocalSystem", dwError);
+ goto done;
+ }
+ cbSid = SECURITY_MAX_SID_SIZE;
+ if (!CreateWellKnownSid(WinLocalSystemSid, NULL, allowedSids[crt], &cbSid)) {
+ dwError = GetLastError();
+ ReportErrorCode(L"CreateWellKnownSid", dwError);
+ goto done;
+ }
+
+ dwError = BuildServiceSecurityDescriptor(JOB_OBJECT_ALL_ACCESS, crt, allowedSids, 0, NULL, pSD);
+ if (ERROR_SUCCESS != dwError) {
+ goto done;
+ }
+
+done:
+ do {
+ if (allowedSids && allowedSids[crt]) LocalFree(allowedSids[crt]);
+ --crt;
+ } while (crt);
+ if (allowedSids) LocalFree(allowedSids);
+ if (value) LocalFree(value);
+
+ return dwError;
+}
+
+
+
+//----------------------------------------------------------------------------
// Function: ValidateImpersonateAccessCheck
//
// Description:
@@ -339,7 +439,8 @@ done:
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
-DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PWSTR cmdLine)
+DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PCWSTR cmdLine,
+ __in SECURITY_DESCRIPTOR* pSdJob)
{
DWORD dwErrorCode = ERROR_SUCCESS;
DWORD exitCode = EXIT_FAILURE;
@@ -350,6 +451,8 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = { 0 };
void * envBlock = NULL;
BOOL createProcessResult = FALSE;
+ SECURITY_ATTRIBUTES saJob;
+ SECURITY_ATTRIBUTES* psaJob = NULL;
wchar_t* curr_dir = NULL;
FILE *stream = NULL;
@@ -357,18 +460,27 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
if (NULL != logonHandle) {
dwErrorCode = ValidateImpersonateAccessCheck(logonHandle);
if (dwErrorCode) {
+ ReportErrorCode(L"ValidateImpersonateAccessCheck", dwErrorCode);
return dwErrorCode;
}
+
+ // We need to create a security descripto for the job so that the task can query it
+ ZeroMemory(&saJob, sizeof(saJob));
+
+ saJob.nLength = sizeof(saJob);
+ saJob.lpSecurityDescriptor = pSdJob;
+ psaJob = &saJob;
}
// Create un-inheritable job object handle and set job object to terminate
// when last handle is closed. So winutils.exe invocation has the only open
// job object handle. Exit of winutils.exe ensures termination of job object.
// Either a clean exit of winutils or crash or external termination.
- jobObject = CreateJobObject(NULL, jobObjName);
+ jobObject = CreateJobObject(psaJob, jobObjName);
dwErrorCode = GetLastError();
if(jobObject == NULL || dwErrorCode == ERROR_ALREADY_EXISTS)
{
+ ReportErrorCode(L"CreateJobObject", dwErrorCode);
return dwErrorCode;
}
jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
@@ -378,6 +490,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
sizeof(jeli)) == 0)
{
dwErrorCode = GetLastError();
+ ReportErrorCode(L"SetInformationJobObject", dwErrorCode);
CloseHandle(jobObject);
return dwErrorCode;
}
@@ -385,6 +498,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
if(AssignProcessToJobObject(jobObject, GetCurrentProcess()) == 0)
{
dwErrorCode = GetLastError();
+ ReportErrorCode(L"AssignProcessToJobObject", dwErrorCode);
CloseHandle(jobObject);
return dwErrorCode;
}
@@ -394,6 +508,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
if(SetEnvironmentVariable(L"JVM_PID", jobObjName) == 0)
{
dwErrorCode = GetLastError();
+ ReportErrorCode(L"SetEnvironmentVariable", dwErrorCode);
// We have to explictly Terminate, passing in the error code
// simply closing the job would kill our own process with success exit status
TerminateJobObject(jobObject, dwErrorCode);
@@ -410,6 +525,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
logonHandle,
TRUE )) {
dwErrorCode = GetLastError();
+ ReportErrorCode(L"CreateEnvironmentBlock", dwErrorCode);
// We have to explictly Terminate, passing in the error code
// simply closing the job would kill our own process with success exit status
TerminateJobObject(jobObject, dwErrorCode);
@@ -427,6 +543,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
if (0 == currDirCnt) {
dwErrorCode = GetLastError();
+ ReportErrorCode(L"GetCurrentDirectory", dwErrorCode);
// We have to explictly Terminate, passing in the error code
// simply closing the job would kill our own process with success exit status
TerminateJobObject(jobObject, dwErrorCode);
@@ -463,6 +580,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
if (FALSE == createProcessResult) {
dwErrorCode = GetLastError();
+ ReportErrorCode(L"CreateProcess/AsUser", dwErrorCode);
if( envBlock != NULL ) {
DestroyEnvironmentBlock( envBlock );
envBlock = NULL;
@@ -477,7 +595,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
CloseHandle(pi.hThread);
- ReportErrorCode(L"CreateTaskImpl", ERROR_SUCCESS);
+ ReportErrorCode(L"CreateTaskImpl", ERROR_SUCCESS);
// Wait until child process exits.
WaitForSingleObject( pi.hProcess, INFINITE );
@@ -525,10 +643,11 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
DWORD CreateTask(__in PCWSTR jobObjName,__in PWSTR cmdLine)
{
// call with null logon in order to create tasks utilizing the current logon
- return CreateTaskImpl( NULL, jobObjName, cmdLine );
+ return CreateTaskImpl( NULL, jobObjName, cmdLine, NULL);
}
+
//----------------------------------------------------------------------------
-// Function: CreateTask
+// Function: CreateTaskAsUser
//
// Description:
// Creates a task via a jobobject. Outputs the
@@ -547,7 +666,7 @@ DWORD CreateTaskAsUser(__in PCWSTR jobObjName,
PROFILEINFO pi;
BOOL profileIsLoaded = FALSE;
FILE* pidFile = NULL;
-
+ SECURITY_DESCRIPTOR* pSdJob = NULL;
DWORD retLen = 0;
HANDLE logonHandle = NULL;
@@ -572,6 +691,12 @@ DWORD CreateTaskAsUser(__in PCWSTR jobObjName,
goto done;
}
+ err = EnablePrivilege(SE_SECURITY_NAME);
+ if( err != ERROR_SUCCESS ) {
+ ReportErrorCode(L"EnablePrivilege:SE_SECURITY_NAME", err);
+ goto done;
+ }
+
err = RegisterWithLsa(LOGON_PROCESS_NAME ,&lsaHandle);
if( err != ERROR_SUCCESS ) {
ReportErrorCode(L"RegisterWithLsa", err);
@@ -617,12 +742,21 @@ DWORD CreateTaskAsUser(__in PCWSTR jobObjName,
fclose(pidFile);
if (err != ERROR_SUCCESS) {
+ ReportErrorCode(L"fprintf_s:pidFilePath", err);
+ goto done;
+ }
+
+ err = BuildJobObjectSecurityDescriptor(user, &pSdJob);
+ if (ERROR_SUCCESS != err) {
+ ReportErrorCode(L"BuildJobSecurityDescriptor", err);
goto done;
}
-
- err = CreateTaskImpl(logonHandle, jobObjName, cmdLine);
+
+ err = CreateTaskImpl(logonHandle, jobObjName, cmdLine, pSdJob);
-done:
+done:
+ if (pSdJob) LocalFree(pSdJob);
+
if( profileIsLoaded ) {
UnloadProfileForLogon( logonHandle, &pi );
profileIsLoaded = FALSE;
@@ -638,7 +772,6 @@ done:
return err;
}
-
//----------------------------------------------------------------------------
// Function: IsTaskAlive
//
@@ -699,11 +832,8 @@ DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob)
return ERROR_SUCCESS;
}
-//----------------------------------------------------------------------------
-// Function: KillTask
-//
-// Description:
-// Kills a task via a jobobject. Outputs the
+//-----------------------------------------------------------------------------
+// Function: KillTask-//-// Description:-// Kills a task via a jobobject. Outputs the
// appropriate information to stdout on success, or stderr on failure.
//
// Returns:
@@ -716,7 +846,7 @@ DWORD KillTask(PCWSTR jobObjName)
{
DWORD err = GetLastError();
if(err == ERROR_FILE_NOT_FOUND)
- {
+ {
// job object does not exist. assume its not alive
return ERROR_SUCCESS;
}
@@ -728,7 +858,6 @@ DWORD KillTask(PCWSTR jobObjName)
return GetLastError();
}
CloseHandle(jobObject);
-
return ERROR_SUCCESS;
}
@@ -958,7 +1087,7 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[])
}
TaskExit:
- ReportErrorCode(L"TaskExit:", dwErrorCode);
+ ReportErrorCode(L"TaskExit:", dwErrorCode);
return dwErrorCode;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 7dbceaf..eed2175 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -87,9 +87,14 @@ public Configuration getConf() {
* @param owner
* @throws IOException
*/
- public void localizeClasspathJar(Path classPathJar, String owner) throws IOException {
- // For the default container this is a no-op
- // The WindowsSecureContainerExecutor overrides this
+ public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner) throws IOException {
+ // None secure executor simply use the classpath create din the NM fprivate folder
+ return classPathJar;
+ }
+
+
+ public Path getContainerClasspathJarPrivateDir(String pwd) throws IOException {
+ return new Path(pwd);
}
/**
@@ -112,7 +117,7 @@ public void localizeClasspathJar(Path classPathJar, String owner) throws IOExcep
*/
public abstract void startLocalizer(Path nmPrivateContainerTokens,
InetSocketAddress nmAddr, String user, String appId, String locId,
- List localDirs, List logDirs)
+ LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException;
@@ -132,8 +137,8 @@ public abstract void startLocalizer(Path nmPrivateContainerTokens,
*/
public abstract int launchContainer(Container container,
Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath,
- String user, String appId, Path containerWorkDir, List localDirs,
- List logDirs) throws IOException;
+ String user, String appId, Path containerWorkDir,
+ List localDirs, List logDirs) throws IOException;
public abstract boolean signalContainer(String user, String pid,
Signal signal)
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index ce66c90..e29244c 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import com.google.common.base.Optional;
+
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
@@ -94,13 +95,16 @@ public void init() throws IOException {
@Override
public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
- List localDirs, List logDirs)
+ LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
+ List localDirs = dirsHandler.getLocalDirs();
+ List logDirs = dirsHandler.getLogDirs();
+
ContainerLocalizer localizer =
new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
RecordFactoryProvider.getRecordFactory(getConf()));
-
+
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId);
@@ -124,7 +128,7 @@ public int launchContainer(Container container,
Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath,
String userName, String appId, Path containerWorkDir,
List localDirs, List logDirs) throws IOException {
-
+
FsPermission dirPerm = new FsPermission(APPDIR_PERM);
ContainerId containerId = container.getContainerId();
@@ -150,16 +154,17 @@ public int launchContainer(Container container,
YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
createDir(tmpDir, dirPerm, false, userName);
- // copy launch script to work dir
- Path launchDst =
- new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
- copyFile(nmPrivateContainerScriptPath, launchDst, userName);
// copy container tokens to work dir
Path tokenDst =
new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE);
copyFile(nmPrivateTokensPath, tokenDst, userName);
+ // copy launch script to work dir
+ Path launchDst =
+ new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
+ copyFile(nmPrivateContainerScriptPath, launchDst, userName);
+
// Create new local launch wrapper script
LocalWrapperScriptBuilder sb = getLocalWrapperScriptBuilder(
containerIdStr, containerWorkDir);
@@ -183,7 +188,7 @@ public int launchContainer(Container container,
+ " was marked as inactive. Returning terminated error");
return ExitCode.TERMINATED.getExitCode();
}
-
+
// create log dir under app
// fork script
Shell.ICommandExecutor shExec = null;
@@ -240,7 +245,7 @@ public int launchContainer(Container container,
}
return exitCode;
} finally {
- shExec.dispose(); //
+ if (null != shExec) shExec.dispose();
}
return 0;
}
@@ -433,7 +438,7 @@ public static boolean containerIsAlive(String pid) throws IOException {
* @param signal signal to send
* (for logging).
*/
- private void killContainer(String pid, Signal signal) throws IOException {
+ protected void killContainer(String pid, Signal signal) throws IOException {
new ShellCommandExecutor(Shell.getSignalKillCommand(signal.getValue(), pid))
.execute();
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index c1183eb..587e958 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import com.google.common.base.Optional;
+
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -182,9 +183,12 @@ public void init() throws IOException {
@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
- List localDirs, List logDirs)
+ LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
+ List localDirs = dirsHandler.getLocalDirs();
+ List logDirs = dirsHandler.getLogDirs();
+
verifyUsernamePattern(user);
String runAsUser = getRunAsUser(user);
List command = new ArrayList();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
index baa0b58..30018ea 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -22,11 +22,11 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.OutputStream;
import java.io.PrintStream;
-import java.io.Reader;
import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -35,27 +35,40 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.WinutilsProcessStub;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ICommandExecutor;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
/**
- * Windows secure container executor. Uses winutils task createAsUser.
- *
+ * Windows secure container executor (WSCE).
+ * This class offers a secure container executor on Windows, similar to the LinuxContainerExecutor
+ * As the NM does not run on a high privileged context, this class delegates elevated operations
+ * to the helper hadoopwintuilsvc, implemented by the winutils.exe running as a service.
+ * JNI and LRPC is used to communicate with the privileged service.
*/
public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
private static final Log LOG = LogFactory
.getLog(WindowsSecureContainerExecutor.class);
+
+ public static final String LOCALIZER_PID_FORMAT = "STAR_LOCALIZER_%s";
+ /**
+ * A shell script wrapper builder for WSCE.
+ * Overwrites the default behavior to remove the creation of the PID file in the script wrapper.
+ * WSCE creates the pid file as part of launching the task in winutils
+ */
private class WindowsSecureWrapperScriptBuilder
extends LocalWrapperScriptBuilder {
@@ -68,6 +81,86 @@ protected void writeLocalWrapperScript(Path launchDst, Path pidFile, PrintStream
pout.format("@call \"%s\"", launchDst);
}
}
+
+ /**
+ * This is a skeleton file system used to elevate certain operations.
+ * WSCE has to create container dirs under local/userchache/$user but
+ * this dir itself is owned by $user, with chmod 750. As ther NM has no
+ * write access, it must delegate the write operations to the privileged
+ * hadoopwintuilsvc.
+ */
+ private static class ElevatedFileSystem extends DelegateToFileSystem {
+
+ /**
+ * This overwrites certain RawLocalSystem operations to be performed by a privileged process.
+ *
+ */
+ private static class ElevatedRawLocalFilesystem extends RawLocalFileSystem {
+
+ @Override
+ protected boolean mkOneDir(File p2f) throws IOException {
+ Path path = new Path(p2f.getAbsolutePath());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:mkOneDir: %s", path));
+ }
+ boolean ret = false;
+
+ // File.mkdir returns false, does not throw. Must mimic it.
+ try {
+ NativeIO.Elevated.mkdir(path);
+ ret = true;
+ }
+ catch(Throwable e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:mkOneDir: %s",
+ org.apache.hadoop.util.StringUtils.stringifyException(e)));
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public void setPermission(Path p, FsPermission permission) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:setPermission: %s %s", p, permission));
+ }
+ //super.setPermission(p, permission);
+ NativeIO.Elevated.chmod(p, permission.toShort());
+ }
+
+ @Override
+ public void setOwner(Path p, String username, String groupname) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:setOwner: %s %s %s", p, username, groupname));
+ }
+ NativeIO.Elevated.chown(p, username, groupname);
+ }
+
+ @Override
+ protected OutputStream createOutputStream(Path f, boolean append) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:create: %s %b", f, append));
+ }
+ return NativeIO.Elevated.create(f, append);
+ }
+
+ @Override
+ public boolean delete(Path p, boolean recursive) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("EFS:delete: %s %b", p, recursive));
+ }
+ return super.delete(p, recursive);
+ }
+ }
+
+ protected ElevatedFileSystem() throws IOException, URISyntaxException {
+ super(FsConstants.LOCAL_FS_URI,
+ new ElevatedRawLocalFilesystem(),
+ new Configuration(),
+ FsConstants.LOCAL_FS_URI.getScheme(),
+ false);
+ }
+ }
private static class WintuilsProcessStubExecutor implements Shell.ICommandExecutor {
private WinutilsProcessStub processStub;
@@ -87,16 +180,13 @@ protected void writeLocalWrapperScript(Path launchDst, Path pidFile, PrintStream
private final String userName;
private final String pidFile;
private final String cmdLine;
- private final Configuration conf;
public WintuilsProcessStubExecutor(
- Configuration conf,
String cwd,
String jobName,
String userName,
String pidFile,
String cmdLine) {
- this.conf = conf;
this.cwd = cwd;
this.jobName = jobName;
this.userName = userName;
@@ -189,6 +279,10 @@ public void dispose() {
}
private String nodeManagerGroup;
+
+ public WindowsSecureContainerExecutor() throws IOException, URISyntaxException {
+ super(FileContext.getFileContext(new ElevatedFileSystem(), new Configuration()));
+ }
@Override
public void setConf(Configuration conf) {
@@ -199,10 +293,14 @@ public void setConf(Configuration conf) {
@Override
protected String[] getRunCommand(String command, String groupId,
String userName, Path pidFile, Configuration conf) {
+ File f = new File(command);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("getRunCommand: %s exists:%b", command, f.exists()));
+ }
return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId, userName,
pidFile.toString(), "cmd /c " + command };
}
-
+
@Override
protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder(
String containerIdStr, Path containerWorkDir) {
@@ -211,51 +309,75 @@ protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder(
@Override
protected void copyFile(Path src, Path dst, String owner) throws IOException {
- super.copyFile(src, dst, owner);
- lfs.setOwner(dst, owner, nodeManagerGroup);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("copyFile: %s -> %s owner:%s", src.toString(), dst.toString(), owner));
+ }
+ NativeIO.Elevated.copy(src, dst, true);
+ NativeIO.Elevated.chown(dst, owner, nodeManagerGroup);
}
@Override
protected void createDir(Path dirPath, FsPermission perms,
boolean createParent, String owner) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("createDir: %s perm:%s owner:%s", dirPath.toString(), perms.toString(), owner));
+ }
+
super.createDir(dirPath, perms, createParent, owner);
lfs.setOwner(dirPath, owner, nodeManagerGroup);
}
@Override
protected void setScriptExecutable(Path script, String owner) throws IOException {
- super.setScriptExecutable(script, null);
- lfs.setOwner(script, owner, nodeManagerGroup);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("setScriptExecutable: %s owner:%s", script.toString(), owner));
+ }
+ super.setScriptExecutable(script, owner);
+ NativeIO.Elevated.chown(script, owner, nodeManagerGroup);
}
@Override
- public void localizeClasspathJar(Path classpathJar, String owner) throws IOException {
- lfs.setOwner(classpathJar, owner, nodeManagerGroup);
+ public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("localizeClasspathJar: %s %s o:%s", classPathJar, pwd, owner));
+ }
+ createDir(pwd, new FsPermission(APPDIR_PERM), true, owner);
+ String fileName = classPathJar.getName();
+ Path dst = new Path(pwd, fileName);
+ NativeIO.Elevated.move(classPathJar, dst, true);
+ NativeIO.Elevated.chown(dst, owner, nodeManagerGroup);
+ return dst;
}
@Override
public void startLocalizer(Path nmPrivateContainerTokens,
InetSocketAddress nmAddr, String user, String appId, String locId,
- List localDirs, List logDirs) throws IOException,
+ LocalDirsHandlerService dirsHandler) throws IOException,
InterruptedException {
-
+
+ List localDirs = dirsHandler.getLocalDirs();
+ List logDirs = dirsHandler.getLogDirs();
+
+ Path classpathJarPrivateDir = dirsHandler.getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR);
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId);
createAppLogDirs(appId, logDirs, user);
+
// TODO: Why pick first app dir. The same in LCE why not random?
Path appStorageDir = getFirstApplicationDir(localDirs, user, appId);
-
+
String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
Path tokenDst = new Path(appStorageDir, tokenFn);
- LOG.info("Copying from " + nmPrivateContainerTokens + " to " + tokenDst);
copyFile(nmPrivateContainerTokens, tokenDst, user);
- List command ;
-
File cwdApp = new File(appStorageDir.toString());
- LOG.info(String.format("cwdApp: %s", cwdApp));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("cwdApp: %s", cwdApp));
+ }
+
+ List command ;
command = new ArrayList();
@@ -268,11 +390,12 @@ public void startLocalizer(Path nmPrivateContainerTokens,
// Passing CLASSPATH explicitly is *way* too long for command line.
String classPath = System.getProperty("java.class.path");
Map env = new HashMap(System.getenv());
- String classPathJar = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
- localizeClasspathJar(new Path(classPathJar), user);
+ String classPathJar = FileUtil.createJarWithClassPath(classPath, classpathJarPrivateDir, env);
+ classPathJar = localizeClasspathJar(
+ new Path(classPathJar), new Path(cwdApp.getPath()), user).toString();
command.add("-classpath");
command.add(classPathJar);
-
+
String javaLibPath = System.getProperty("java.library.path");
if (javaLibPath != null) {
command.add("-Djava.library.path=" + javaLibPath);
@@ -282,16 +405,25 @@ public void startLocalizer(Path nmPrivateContainerTokens,
String cmdLine = StringUtils.join(command, " ");
+ String localizerPid = String.format(LOCALIZER_PID_FORMAT, locId);
+
WintuilsProcessStubExecutor stubExecutor = new WintuilsProcessStubExecutor(
- getConf(),
cwdApp.getAbsolutePath(),
- "START_LOCALIZER_" + locId, user, "nul:", cmdLine);
+ localizerPid, user, "nul:", cmdLine);
try {
stubExecutor.execute();
stubExecutor.validateResult();
}
finally {
stubExecutor.dispose();
+ try
+ {
+ killContainer(localizerPid, Signal.KILL);
+ }
+ catch(Throwable e) {
+ LOG.warn(String.format("An exception occured during the cleanup of localizer job %s:\n%s",
+ localizerPid, org.apache.hadoop.util.StringUtils.stringifyException(e)));
+ }
}
}
@@ -299,11 +431,10 @@ public void startLocalizer(Path nmPrivateContainerTokens,
protected ICommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr,
String userName, Path pidFile, Configuration conf, File wordDir, Map environment)
throws IOException {
-
+
return new WintuilsProcessStubExecutor(
- getConf(),
- wordDir.toString(),
+ wordDir.toString(),
containerIdStr, userName, pidFile.toString(), "cmd /c " + wrapperScriptPath);
- }
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 87a36c4..ce97dbc 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -212,7 +212,9 @@ public Integer call() {
+ Path.SEPARATOR
+ String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
containerIdStr));
-
+ Path nmPrivateClasspathJarDir =
+ dirsHandler.getLocalPathForWrite(
+ getContainerPrivateDir(appIdStr, containerIdStr));
DataOutputStream containerScriptOutStream = null;
DataOutputStream tokensOutStream = null;
@@ -263,7 +265,7 @@ public Integer call() {
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
- localResources);
+ localResources, nmPrivateClasspathJarDir);
// Write out the environment
writeLaunchEnv(containerScriptOutStream, environment, localResources,
@@ -658,7 +660,8 @@ private static void putEnvIfAbsent(
public void sanitizeEnv(Map environment, Path pwd,
List appDirs, List containerLogDirs,
- Map> resources) throws IOException {
+ Map> resources,
+ Path nmPrivateClasspathJarDir) throws IOException {
/**
* Non-modifiable environment variables
*/
@@ -722,6 +725,7 @@ public void sanitizeEnv(Map environment, Path pwd,
// TODO: Remove Windows check and use this approach on all platforms after
// additional testing. See YARN-358.
if (Shell.WINDOWS) {
+
String inputClassPath = environment.get(Environment.CLASSPATH.name());
if (inputClassPath != null && !inputClassPath.isEmpty()) {
StringBuilder newClassPath = new StringBuilder(inputClassPath);
@@ -765,10 +769,10 @@ public void sanitizeEnv(Map environment, Path pwd,
mergedEnv.putAll(environment);
String classPathJar = FileUtil.createJarWithClassPath(
- newClassPath.toString(), pwd, mergedEnv);
+ newClassPath.toString(), nmPrivateClasspathJarDir, mergedEnv);
// In a secure cluster the classpath jar must be localized to grant access
- this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser());
- environment.put(Environment.CLASSPATH.name(), classPathJar);
+ Path localizedClassPathJar = exec.localizeClasspathJar(new Path(classPathJar), pwd, container.getUser());
+ environment.put(Environment.CLASSPATH.name(), localizedClassPathJar.toString());
}
}
// put AuxiliaryService data to environment
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 64a0b37..36b4578 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1071,15 +1071,14 @@ public void run() {
// 1) write credentials to private dir
writeCredentials(nmPrivateCTokensPath);
// 2) exec initApplication and wait
- List localDirs = dirsHandler.getLocalDirs();
- List logDirs = dirsHandler.getLogDirs();
if (dirsHandler.areDisksHealthy()) {
exec.startLocalizer(nmPrivateCTokensPath, localizationServerAddress,
context.getUser(),
ConverterUtils.toString(
context.getContainerId().
getApplicationAttemptId().getApplicationId()),
- localizerId, localDirs, logDirs);
+ localizerId,
+ dirsHandler);
} else {
throw new IOException("All disks failed. "
+ dirsHandler.getDisksHealthReport());
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 2e9e8b1..d54367a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -185,7 +185,7 @@ public void testStartLocalizer() throws IOException {
Path nmPrivateCTokensPath= new Path("file:///bin/nmPrivateCTokensPath");
try {
- mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler.getLocalDirs(), dirsHandler.getLogDirs());
+ mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler);
List result=readMockParams();
Assert.assertEquals(result.size(), 17);
Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index ed59ddd..187c20b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -737,7 +737,7 @@ public boolean matches(Object o) {
ArgumentCaptor tokenPathCaptor = ArgumentCaptor.forClass(Path.class);
verify(exec).startLocalizer(tokenPathCaptor.capture(),
isA(InetSocketAddress.class), eq("user0"), eq(appStr), eq(ctnrStr),
- isA(List.class), isA(List.class));
+ isA(LocalDirsHandlerService.class));
Path localizationTokenPath = tokenPathCaptor.getValue();
// heartbeat from localizer