commit 2233061cb5e7290be1726152cf6cecc2771a34fa Author: Eric Yang Date: Mon Oct 15 18:47:50 2018 -0400 YARN-8778. Added CLI to shell into Docker container. Contributed by Eric Yang diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 70af1ed..ebb9792 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -786,7 +786,19 @@ org.eclipse.jetty.websocket:javax-websocket-server-impl - * + */** + + + + org.eclipse.jetty.websocket:websocket-client + + */** + + + + org.eclipse.jetty:jetty-io + + */** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 2cb3716..e9548c5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.ShellContainerCommand; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; @@ -560,4 +561,10 @@ public Resource getResourceProfile(String profile) Set hostNames) throws YarnException, IOException { return client.getNodeToAttributes(hostNames); } + + @Override + public void shellToContainer(ContainerId containerId, + ShellContainerCommand command) throws IOException { + new IOException("Operation is not supported."); + } } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 4cdbcfb..83f43c8 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -158,6 +158,7 @@ 1.16 1.2.6 2.0.0-beta-1 + 3.0.0.M1 @@ -821,6 +822,11 @@ + org.eclipse.jetty.websocket + websocket-client + ${jetty.version} + + javax.servlet.jsp jsp-api 2.1 @@ -1301,6 +1307,11 @@ + jline + jline + ${jline.version} + + org.hsqldb hsqldb ${hsqldb.version} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java new file mode 100644 index 0000000..07acb9d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java @@ -0,0 +1,32 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; + +/** + * Enumeration of various signal container commands. + */ +@Public +@Evolving +public enum ShellContainerCommand { + BASH, + SH +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index 851acbd..8d0d89e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.Component; @@ -93,54 +94,6 @@ } /** - * Generate SPNEGO challenge request token. - * - * @param server - hostname to contact - * @throws IOException - * @throws InterruptedException - */ - String generateToken(String server) throws IOException, InterruptedException { - UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); - LOG.debug("The user credential is {}", currentUser); - String challenge = currentUser - .doAs(new PrivilegedExceptionAction() { - @Override - public String run() throws Exception { - try { - // This Oid for Kerberos GSS-API mechanism. - Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID"); - GSSManager manager = GSSManager.getInstance(); - // GSS name for server - GSSName serverName = manager.createName("HTTP@" + server, - GSSName.NT_HOSTBASED_SERVICE); - // Create a GSSContext for authentication with the service. - // We're passing client credentials as null since we want them to - // be read from the Subject. - GSSContext gssContext = manager.createContext( - serverName.canonicalize(mechOid), mechOid, null, - GSSContext.DEFAULT_LIFETIME); - gssContext.requestMutualAuth(true); - gssContext.requestCredDeleg(true); - // Establish context - byte[] inToken = new byte[0]; - byte[] outToken = gssContext.initSecContext(inToken, 0, - inToken.length); - gssContext.dispose(); - // Base64 encoded and stringified token for server - LOG.debug("Got valid challenge for host {}", serverName); - return new String(BASE_64_CODEC.encode(outToken), - StandardCharsets.US_ASCII); - } catch (GSSException | IllegalAccessException - | NoSuchFieldException | ClassNotFoundException e) { - LOG.error("Error: {}", e); - throw new AuthenticationException(e); - } - } - }); - return challenge; - } - - /** * Calculate Resource Manager address base on working REST API. */ String getRMWebAddress() { @@ -177,7 +130,7 @@ String getRMWebAddress() { .resource(sb.toString()).type(MediaType.APPLICATION_JSON); if (useKerberos) { String[] server = host.split(":"); - String challenge = generateToken(server[0]); + String challenge = YarnClientUtils.generateToken(server[0]); builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + challenge); LOG.debug("Authorization: Negotiate {}", challenge); @@ -289,7 +242,7 @@ private Builder getApiClient(String requestPath) if (conf.get("hadoop.http.authentication.type").equals("kerberos")) { try { URI url = new URI(requestPath); - String challenge = generateToken(url.getHost()); + String challenge = YarnClientUtils.generateToken(url.getHost()); builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + challenge); } catch (Exception e) { throw new IOException(e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java index f955064..1ec8d41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java @@ -41,6 +41,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.log4j.Logger; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; @@ -169,8 +170,7 @@ public void tearDown() throws Exception { public void testHttpSpnegoChallenge() throws Exception { UserGroupInformation.loginUserFromKeytab(clientPrincipal, keytabFile .getCanonicalPath()); - asc = new ApiServiceClient(); - String challenge = asc.generateToken("localhost"); + String challenge = YarnClientUtils.generateToken("localhost"); assertNotNull(challenge); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 2e0c777..3878ba3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -54,6 +54,10 @@ log4j log4j + + org.eclipse.jetty.websocket + websocket-client + @@ -127,6 +131,10 @@ test-jar + + jline + jline + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java new file mode 100644 index 0000000..cfb596b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.jline.terminal.Terminal; +import org.jline.terminal.TerminalBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Web socket for establishing interactive command shell connection through + * Node Manage to container executor. + */ +@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce", "YARN" }) +@InterfaceStability.Unstable + +@WebSocket +public class ContainerShellWebSocket { + private static final Logger LOG = + LoggerFactory.getLogger(ContainerShellWebSocket.class); + + private Session session; + private Terminal terminal; + + @OnWebSocketMessage + public void onText(Session session, String message) throws IOException { + terminal.output().write(message.getBytes(Charset.forName("UTF-8"))); + } + + @OnWebSocketConnect + public void onConnect(Session session) { + try { + this.session = session; + try { + terminal = TerminalBuilder.builder() + .system(true) + .build(); + } catch (IOException t) { + terminal = TerminalBuilder.builder() + .system(false) + .streams(System.in, (OutputStream) System.out) + .build(); + } + LOG.info(session.getRemoteAddress().getHostString() + " connected!"); + } catch (IOException e) { + session.close(1002, e.getMessage()); + } + } + + @OnWebSocketClose + public void onClose(Session session, int status, String reason) { + if (status==1000) { + LOG.info(session.getRemoteAddress().getHostString() + + " closed, status: " + status); + } else { + LOG.error(session.getRemoteAddress().getHostString() + + " closed, status: " + status + " Error: " + reason); + } + } + + public void run() { + try { + while (session.isOpen()) { + if ( terminal.input().available() != 0 ) { + int c = terminal.input().read(); + if ( c == 0x1B ) { + break; + } + String formatted = Character.toString( (char) c); + session.getRemote().sendString(formatted); + } + } + } catch (IOException e) { + try { + session.disconnect(); + } catch (IOException e1) { + LOG.error("Error closing connection: ", e1); + } + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 59fa6a8..f7699b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.ShellContainerCommand; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -958,4 +959,18 @@ public abstract Resource getResourceProfile(String profile) public abstract Map> getNodeToAttributes( Set hostNames) throws YarnException, IOException; + /** + *

+ * The interface used by client to get a shell to a container. + *

+ * + * @param containerId + * @param command + * @throws IOException + */ + @Public + @Unstable + public abstract void shellToContainer(ContainerId containerId, + ShellContainerCommand command) throws IOException; + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index acfc3ff..28402f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; +import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.EnumSet; @@ -27,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.Future; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -111,15 +113,18 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.ShellContainerCommand; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.api.AHSClient; +import org.apache.hadoop.yarn.client.api.ContainerShellWebSocket; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; @@ -132,6 +137,9 @@ import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; +import org.eclipse.jetty.websocket.client.WebSocketClient; import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; @@ -1006,4 +1014,43 @@ public Resource getResourceProfile(String profile) GetNodesToAttributesRequest.newInstance(hostNames); return rmClient.getNodesToAttributes(request).getNodeToAttributes(); } + + @Override + public void shellToContainer(ContainerId containerId, + ShellContainerCommand command) throws IOException { + try { + GetContainerReportRequest request = Records + .newRecord(GetContainerReportRequest.class); + request.setContainerId(containerId); + GetContainerReportResponse response = rmClient + .getContainerReport(request); + String host = new URI(response.getContainerReport() + .getNodeHttpAddress()).getHost(); + WebSocketClient client = new WebSocketClient(); + URI uri = URI.create("ws://" + host + ":8042/container/"+containerId); + try { + client.start(); + // The socket that receives events + ContainerShellWebSocket socket = new ContainerShellWebSocket(); + ClientUpgradeRequest upgradeRequest = new ClientUpgradeRequest(); + if (UserGroupInformation.isSecurityEnabled()) { + String challenge = YarnClientUtils.generateToken(host); + upgradeRequest.setHeader("Authorization", "Negotiate " + challenge); + } + // Attempt Connect + Future fut = client.connect(socket, uri, upgradeRequest); + // Wait for Connect + Session session = fut.get(); + // Send a message + session.getRemote().sendString("Hello"); + socket.run(); + // Close session + session.close(); + } finally { + client.stop(); + } + } catch (Throwable t) { + LOG.error("Fail to shell to container: ", t); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index b0e12bc..184c2d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ShellContainerCommand; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.AppAdminClient; @@ -110,6 +111,7 @@ public static final String COMPONENTS = "components"; public static final String VERSION = "version"; public static final String STATES = "states"; + public static final String SHELL_CMD = "shell"; private static String firstArg = null; @@ -301,6 +303,8 @@ public int run(String[] args) throws Exception { opts.getOption(LIST_CMD).setArgName("Application ID"); opts.getOption(FAIL_CMD).setArgName("Application Attempt ID"); } else if (title != null && title.equalsIgnoreCase(CONTAINER)) { + opts.addOption(SHELL_CMD, true, + "Run a shell in the container."); opts.addOption(STATUS_CMD, true, "Prints the status of the container."); opts.addOption(LIST_CMD, true, @@ -313,6 +317,7 @@ public int run(String[] args) throws Exception { "app version, -components to filter instances based on component " + "names, -states to filter instances based on instance state."); opts.addOption(HELP_CMD, false, "Displays help for all commands."); + opts.getOption(SHELL_CMD).setArgName("Container ID"); opts.getOption(STATUS_CMD).setArgName("Container ID"); opts.getOption(LIST_CMD).setArgName("Application Name or Attempt ID"); opts.addOption(APP_TYPE_CMD, true, "Works with -list to " + @@ -542,6 +547,19 @@ public int run(String[] args) throws Exception { command = SignalContainerCommand.valueOf(signalArgs[1]); } signalToContainer(containerId, command); + } else if (cliParser.hasOption(SHELL_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, SHELL_CMD)) { + printUsage(title, opts); + return exitCode; + } + final String[] shellArgs = cliParser.getOptionValues(SHELL_CMD); + final String containerId = shellArgs[0]; + ShellContainerCommand command = + ShellContainerCommand.BASH; + if (shellArgs.length == 2) { + command = ShellContainerCommand.valueOf(shellArgs[1]); + } + shellToContainer(containerId, command); } else if (cliParser.hasOption(LAUNCH_CMD)) { if (hasAnyOtherCLIOptions(cliParser, opts, LAUNCH_CMD, APP_TYPE_CMD, UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) { @@ -798,6 +816,20 @@ private void signalToContainer(String containerIdStr, } /** + * Shell to the containerId + * + * @param containerIdStr the container id + * @param command the shell command + * @throws YarnException + */ + private void shellToContainer(String containerIdStr, + ShellContainerCommand command) throws YarnException, IOException { + ContainerId containerId = ContainerId.fromString(containerIdStr); + sysout.println("Shelling to container " + containerIdStr); + client.shellToContainer(containerId, command); + } + + /** * It prints the usage of the command * * @param opts diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java index 1717675..2fe5936 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java @@ -19,15 +19,29 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; import com.google.common.collect.ImmutableSet; + +import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.ietf.jgss.GSSContext; +import org.ietf.jgss.GSSException; +import org.ietf.jgss.GSSManager; +import org.ietf.jgss.GSSName; +import org.ietf.jgss.Oid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class is a container for utility methods that are useful when creating @@ -35,6 +49,9 @@ */ public abstract class YarnClientUtils { + private static final Logger LOG = + LoggerFactory.getLogger(YarnClientUtils.class); + private static final Base64 BASE_64_CODEC = new Base64(0); private static final String ADD_LABEL_FORMAT_ERR_MSG = "Input format for adding node-labels is not correct, it should be " + "labelName1[(exclusive=true/false)],LabelName2[] .."; @@ -187,4 +204,52 @@ static YarnConfiguration getYarnConfWithRmHaId(Configuration conf) return yarnConf; } + + /** + * Generate SPNEGO challenge request token. + * + * @param server - hostname to contact + * @throws IOException + * @throws InterruptedException + */ + public static String generateToken(String server) throws IOException, InterruptedException { + UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); + LOG.debug("The user credential is {}", currentUser); + String challenge = currentUser + .doAs(new PrivilegedExceptionAction() { + @Override + public String run() throws Exception { + try { + // This Oid for Kerberos GSS-API mechanism. + Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID"); + GSSManager manager = GSSManager.getInstance(); + // GSS name for server + GSSName serverName = manager.createName("HTTP@" + server, + GSSName.NT_HOSTBASED_SERVICE); + // Create a GSSContext for authentication with the service. + // We're passing client credentials as null since we want them to + // be read from the Subject. + GSSContext gssContext = manager.createContext( + serverName.canonicalize(mechOid), mechOid, null, + GSSContext.DEFAULT_LIFETIME); + gssContext.requestMutualAuth(true); + gssContext.requestCredDeleg(true); + // Establish context + byte[] inToken = new byte[0]; + byte[] outToken = gssContext.initSecContext(inToken, 0, + inToken.length); + gssContext.dispose(); + // Base64 encoded and stringified token for server + LOG.debug("Got valid challenge for host {}", serverName); + return new String(BASE_64_CODEC.encode(outToken), + StandardCharsets.US_ASCII); + } catch (GSSException | IllegalAccessException + | NoSuchFieldException | ClassNotFoundException e) { + LOG.error("Error: {}", e); + throw new AuthenticationException(e); + } + } + }); + return challenge; + } }