From 91d8ec2110d43d6abba39a27e894b151483ce4b7 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Sun, 18 Aug 2019 12:17:31 +0530 Subject: [PATCH] YARN-9728. Skip Invalid XML Chars in Diagnostics field of RM REST Api apps. --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++ .../src/main/resources/yarn-default.xml | 9 +++ .../resourcemanager/webapp/RMWebServices.java | 9 ++- .../server/resourcemanager/webapp/dao/AppInfo.java | 40 ++++++++++- .../resourcemanager/webapp/TestRMWebServices.java | 79 ++++++++++++++++++++++ 5 files changed, 138 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 134b698..0c75dc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3914,6 +3914,10 @@ public static boolean areNodeLabelsEnabled( public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER = false; + public static final String FILTER_INVALID_XML_CHARS = + "yarn.webapp.filter-invalid-xml-chars"; + public static final boolean DEFAULT_FILTER_INVALID_XML_CHARS = false; + // RM and NM CSRF props public static final String REST_CSRF = "webapp.rest-csrf."; public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 4b93d1e..3398cb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3781,6 +3781,15 @@ + yarn.webapp.filter-invalid-xml-chars + false + + Flag to enable filter of invalid xml characters present in the + value of diagnostics field of apps output from RM WebService. + + + + The type of configuration store to use for scheduler configurations. Default is "file", which uses file based capacity-scheduler.xml to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 762569f..63dfc95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -241,6 +241,7 @@ @VisibleForTesting boolean isCentralizedNodeLabelConfiguration = true; private boolean filterAppsByUser = false; + private boolean filterInvalidXMLChars = false; public final static String DELEGATION_TOKEN_HEADER = "Hadoop-YARN-RM-Delegation-Token"; @@ -256,6 +257,9 @@ public RMWebServices(final ResourceManager rm, Configuration conf) { this.filterAppsByUser = conf.getBoolean( YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + this.filterInvalidXMLChars = conf.getBoolean( + YarnConfiguration.FILTER_INVALID_XML_CHARS, + YarnConfiguration.DEFAULT_FILTER_INVALID_XML_CHARS); } RMWebServices(ResourceManager rm, Configuration conf, @@ -625,7 +629,8 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, } AppInfo app = new AppInfo(rm, rmapp, allowAccess, - WebAppUtils.getHttpSchemePrefix(conf), deSelectFields); + WebAppUtils.getHttpSchemePrefix(conf), deSelectFields, + filterInvalidXMLChars); allApps.add(app); } return allApps; @@ -957,7 +962,7 @@ public AppInfo getApp(@Context HttpServletRequest hsr, deSelectFields.initFields(unselectedFields); return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://", - deSelectFields); + deSelectFields, filterInvalidXMLChars); } @GET diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 63b6fe0..566d599 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -136,9 +136,15 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this(rm, app, hasAccess, schemePrefix, new DeSelectFields()); } - @SuppressWarnings({ "rawtypes", "unchecked" }) public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, String schemePrefix, DeSelectFields deSelects) { + this(rm, app, hasAccess, schemePrefix, deSelects, false); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, + String schemePrefix, DeSelectFields deSelects, + Boolean filterInvalidXMLChars) { this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); @@ -174,7 +180,12 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.diagnostics = app.getDiagnostics().toString(); if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; + } else { + if (filterInvalidXMLChars) { + this.diagnostics = stripInValidXMLCharacters(this.diagnostics); + } } + if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { this.applicationTags = Joiner.on(',').join(app.getApplicationTags()); @@ -346,6 +357,33 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, } } + /** + * This method ensures that the output String has only + * valid XML unicode characters as specified by the + * XML 1.0 standard. For reference, please see + * + * the standard. + * + * @param in The String whose non-valid characters we want to remove. + * @return The in String, stripped of non-valid characters. + */ + public static String stripInValidXMLCharacters(String in) { + StringBuffer out = new StringBuffer(); + char current; + + for (int i = 0; i < in.length(); i++) { + current = in.charAt(i); + if ((current == 0x9) || + (current == 0xA) || + (current == 0xD) || + ((current >= 0x20) && (current <= 0xD7FF)) || + ((current >= 0xE000) && (current <= 0xFFFD))) { + out.append(current); + } + } + return out.toString(); + } + public boolean isTrackingUrlReady() { return !this.trackingUrlIsNotReady; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 9e9fe6e..577fcd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -30,8 +31,13 @@ import java.io.StringReader; import java.security.Principal; import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Set; +import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -49,12 +55,21 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.*; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -870,6 +885,70 @@ public void testClusterUserInfo() throws JSONException, Exception { verifyClusterUserInfo(userInfo, "yarn", "admin"); } + @Test + public void testInvalidXMLChars() throws Exception { + ResourceManager mockRM = mock(ResourceManager.class); + + ApplicationId applicationId = ApplicationId.newInstance(1234, 5); + ApplicationReport appReport = ApplicationReport.newInstance( + applicationId, ApplicationAttemptId.newInstance(applicationId, 1), + "user", "queue", "appname", "host", 124, null, + YarnApplicationState.FAILED, "java.lang.Exception: \u0001", "url", + 0, 0, 0, FinalApplicationStatus.FAILED, null, "N/A", 0.53789f, "YARN", + null, null, false, Priority.newInstance(0), "high-mem", "high-mem"); + List appReports = new ArrayList(); + appReports.add(appReport); + + GetApplicationsResponse response = mock(GetApplicationsResponse.class); + when(response.getApplicationList()).thenReturn(appReports); + ClientRMService clientRMService = mock(ClientRMService.class); + when(clientRMService.getApplications(any(GetApplicationsRequest.class))) + .thenReturn(response); + when(mockRM.getClientRMService()).thenReturn(clientRMService); + + RMContext rmContext = mock(RMContext.class); + when(rmContext.getDispatcher()).thenReturn(mock(Dispatcher.class)); + + ApplicationSubmissionContext applicationSubmissionContext = mock( + ApplicationSubmissionContext.class); + when(applicationSubmissionContext.getUnmanagedAM()).thenReturn(true); + + RMApp app = mock(RMApp.class); + RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), + 0, 0, new HashMap<>(), new HashMap<>()); + when(app.getDiagnostics()).thenReturn( + new StringBuilder("java.lang.Exception: \u0001")); + when(app.getApplicationId()).thenReturn(applicationId); + when(app.getUser()).thenReturn("user"); + when(app.getName()).thenReturn("appname"); + when(app.getQueue()).thenReturn("queue"); + when(app.getRMAppMetrics()).thenReturn(appMetrics); + when(app.getApplicationSubmissionContext()).thenReturn( + applicationSubmissionContext); + + ConcurrentMap applications = + new ConcurrentHashMap<>(); + applications.put(applicationId, app); + + when(rmContext.getRMApps()).thenReturn(applications); + when(mockRM.getRMContext()).thenReturn(rmContext); + + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.FILTER_INVALID_XML_CHARS, true); + RMWebServices webSvc = new RMWebServices(mockRM, conf, mock( + HttpServletResponse.class)); + + HttpServletRequest mockHsr = mock(HttpServletRequest.class); + Set emptySet = Collections.unmodifiableSet(Collections.emptySet()); + + AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, + null, null, null, null, null, null, null, emptySet, emptySet, null); + + assertEquals("Incorrect Number of Apps", 1, appsInfo.getApps().size()); + assertEquals("Invalid XML Characters Present", "java.lang.Exception: ", + appsInfo.getApps().get(0).getNote()); + } + public void verifyClusterUserInfo(ClusterUserInfo userInfo, String rmLoginUser, String requestedUser) { assertEquals("rmLoginUser doesn't match: ", -- 2.7.4 (Apple Git-66)