From b6e4310b51cd57fc70b021e9a42bf496e912a0e0 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 6 Sep 2019 11:33:00 +0530 Subject: [PATCH] YARN-9728. Skip Invalid XML Chars in Diagnostics field of RM REST Api apps. --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++ .../src/main/resources/yarn-default.xml | 9 +++ .../resourcemanager/webapp/RMWebServices.java | 61 +++++++++++++++- .../server/resourcemanager/webapp/dao/AppInfo.java | 4 ++ .../resourcemanager/webapp/TestRMWebServices.java | 82 ++++++++++++++++++++++ 5 files changed, 158 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0f1c544..1e525cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3942,6 +3942,10 @@ public static boolean areNodeLabelsEnabled( public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER = false; + public static final String FILTER_INVALID_XML_CHARS = + "yarn.webapp.filter-invalid-xml-chars"; + public static final boolean DEFAULT_FILTER_INVALID_XML_CHARS = false; + // RM and NM CSRF props public static final String REST_CSRF = "webapp.rest-csrf."; public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e2be568..1ff6ca9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3794,6 +3794,15 @@ + yarn.webapp.filter-invalid-xml-chars + false + + Flag to enable filter of invalid xml 1.0 characters present in the + value of diagnostics field of apps output from RM WebService. + + + + The type of configuration store to use for scheduler configurations. Default is "file", which uses file based capacity-scheduler.xml to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index aa29ee6..6a413d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -242,6 +242,7 @@ @VisibleForTesting boolean isCentralizedNodeLabelConfiguration = true; private boolean filterAppsByUser = false; + private boolean filterInvalidXMLChars = false; public final static String DELEGATION_TOKEN_HEADER = "Hadoop-YARN-RM-Delegation-Token"; @@ -257,6 +258,9 @@ public RMWebServices(final ResourceManager rm, Configuration conf) { this.filterAppsByUser = conf.getBoolean( YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + this.filterInvalidXMLChars = conf.getBoolean( + YarnConfiguration.FILTER_INVALID_XML_CHARS, + YarnConfiguration.DEFAULT_FILTER_INVALID_XML_CHARS); } RMWebServices(ResourceManager rm, Configuration conf, @@ -551,6 +555,38 @@ private RMNode getRMNode(final String nodeId) { return ni; } + /** + * This method ensures that the output String has only + * valid XML unicode characters as specified by the + * XML 1.0 standard. For reference, please see + * + * the standard. + * + * @param str The String whose invalid xml characters we want to escape. + * @return The str String after escaping invalid xml characters. + */ + public static String escapeInvalidXMLCharacters(String str) { + StringBuffer out = new StringBuffer(); + final int strlen = str.length(); + final String substitute = "\uFFFD"; + int idx = 0; + while (idx < strlen) { + final int cpt = str.codePointAt(idx); + idx += Character.isSupplementaryCodePoint(cpt) ? 2 : 1; + if ((cpt == 0x9) || + (cpt == 0xA) || + (cpt == 0xD) || + ((cpt >= 0x20) && (cpt <= 0xD7FF)) || + ((cpt >= 0xE000) && (cpt <= 0xFFFD)) || + ((cpt >= 0x10000) && (cpt <= 0x10FFFF))) { + out.append(Character.toChars(cpt)); + } else { + out.append(substitute); + } + } + return out.toString(); + } + @GET @Path(RMWSConsts.APPS) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, @@ -629,6 +665,17 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, WebAppUtils.getHttpSchemePrefix(conf), deSelectFields); allApps.add(app); } + + if (filterInvalidXMLChars) { + final String format = hsr.getHeader(HttpHeaders.ACCEPT); + if (format != null && + format.toLowerCase().contains(MediaType.APPLICATION_XML)) { + for (AppInfo appInfo : allApps.getApps()) { + appInfo.setNote(escapeInvalidXMLCharacters(appInfo.getNote())); + } + } + } + return allApps; } @@ -985,8 +1032,18 @@ public AppInfo getApp(@Context HttpServletRequest hsr, DeSelectFields deSelectFields = new DeSelectFields(); deSelectFields.initFields(unselectedFields); - return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://", - deSelectFields); + AppInfo appInfo = new AppInfo(rm, app, hasAccess(app, hsr), + hsr.getScheme() + "://", deSelectFields); + + if (filterInvalidXMLChars) { + final String format = hsr.getHeader(HttpHeaders.ACCEPT); + if (format != null && + format.toLowerCase().contains(MediaType.APPLICATION_XML)) { + appInfo.setNote(escapeInvalidXMLCharacters(appInfo.getNote())); + } + } + + return appInfo; } @GET diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 63b6fe0..a1d701a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -390,6 +390,10 @@ public String getNote() { return this.diagnostics; } + public void setNote(String diagnosticsMsg) { + this.diagnostics = diagnosticsMsg; + } + public FinalApplicationStatus getFinalStatus() { return this.finalStatus; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 9e9fe6e..d9d1d9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -30,11 +31,17 @@ import java.io.StringReader; import java.security.Principal; import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Set; +import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -49,12 +56,21 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.*; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -870,6 +886,72 @@ public void testClusterUserInfo() throws JSONException, Exception { verifyClusterUserInfo(userInfo, "yarn", "admin"); } + @Test + public void testInvalidXMLChars() throws Exception { + ResourceManager mockRM = mock(ResourceManager.class); + + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationReport appReport = ApplicationReport.newInstance( + applicationId, ApplicationAttemptId.newInstance(applicationId, 1), + "user", "queue", "appname", "host", 124, null, + YarnApplicationState.FAILED, "java.lang.Exception: \u0001", "url", + 0, 0, 0, FinalApplicationStatus.FAILED, null, "N/A", 0.53789f, "YARN", + null, null, false, Priority.newInstance(0), "high-mem", "high-mem"); + List appReports = new ArrayList(); + appReports.add(appReport); + + GetApplicationsResponse response = mock(GetApplicationsResponse.class); + when(response.getApplicationList()).thenReturn(appReports); + ClientRMService clientRMService = mock(ClientRMService.class); + when(clientRMService.getApplications(any(GetApplicationsRequest.class))) + .thenReturn(response); + when(mockRM.getClientRMService()).thenReturn(clientRMService); + + RMContext rmContext = mock(RMContext.class); + when(rmContext.getDispatcher()).thenReturn(mock(Dispatcher.class)); + + ApplicationSubmissionContext applicationSubmissionContext = mock( + ApplicationSubmissionContext.class); + when(applicationSubmissionContext.getUnmanagedAM()).thenReturn(true); + + RMApp app = mock(RMApp.class); + RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), + 0, 0, new HashMap<>(), new HashMap<>()); + when(app.getDiagnostics()).thenReturn( + new StringBuilder("java.lang.Exception: \u0001")); + when(app.getApplicationId()).thenReturn(applicationId); + when(app.getUser()).thenReturn("user"); + when(app.getName()).thenReturn("appname"); + when(app.getQueue()).thenReturn("queue"); + when(app.getRMAppMetrics()).thenReturn(appMetrics); + when(app.getApplicationSubmissionContext()).thenReturn( + applicationSubmissionContext); + + ConcurrentMap applications = + new ConcurrentHashMap<>(); + applications.put(applicationId, app); + + when(rmContext.getRMApps()).thenReturn(applications); + when(mockRM.getRMContext()).thenReturn(rmContext); + + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.FILTER_INVALID_XML_CHARS, true); + RMWebServices webSvc = new RMWebServices(mockRM, conf, mock( + HttpServletResponse.class)); + + HttpServletRequest mockHsr = mock(HttpServletRequest.class); + when(mockHsr.getHeader(HttpHeaders.ACCEPT)). + thenReturn(MediaType.APPLICATION_XML); + Set emptySet = Collections.unmodifiableSet(Collections.emptySet()); + + AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, + null, null, null, null, null, null, null, emptySet, emptySet, null); + + assertEquals("Incorrect Number of Apps", 1, appsInfo.getApps().size()); + assertEquals("Invalid XML Characters Present", + "java.lang.Exception: \uFFFD", appsInfo.getApps().get(0).getNote()); + } + public void verifyClusterUserInfo(ClusterUserInfo userInfo, String rmLoginUser, String requestedUser) { assertEquals("rmLoginUser doesn't match: ", -- 2.7.4 (Apple Git-66)