From 1ace9f9969c681b7bbaaf749b9cb207dfdaf29a6 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Sat, 17 Aug 2019 00:41:26 +0530 Subject: [PATCH] YARN-9728. Skip Invalid XML Chars in Diagnostics field of RM Rest Api apps. --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 +++ .../src/main/resources/yarn-default.xml | 9 +++++ .../resourcemanager/webapp/RMWebServices.java | 9 +++-- .../server/resourcemanager/webapp/dao/AppInfo.java | 40 +++++++++++++++++++++- 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 134b698..0c75dc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3914,6 +3914,10 @@ public static boolean areNodeLabelsEnabled( public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER = false; + public static final String FILTER_INVALID_XML_CHARS = + "yarn.webapp.filter-invalid-xml-chars"; + public static final boolean DEFAULT_FILTER_INVALID_XML_CHARS = false; + // RM and NM CSRF props public static final String REST_CSRF = "webapp.rest-csrf."; public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 4b93d1e..8a7128f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3781,6 +3781,15 @@ + yarn.webapp.filter-invalid-xml-charsr + false + + Flag to enable filter of invalid xml characters present in the + value of diagnostics field of apps output from RM WebService. + + + + The type of configuration store to use for scheduler configurations. Default is "file", which uses file based capacity-scheduler.xml to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 762569f..63dfc95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -241,6 +241,7 @@ @VisibleForTesting boolean isCentralizedNodeLabelConfiguration = true; private boolean filterAppsByUser = false; + private boolean filterInvalidXMLChars = false; public final static String DELEGATION_TOKEN_HEADER = "Hadoop-YARN-RM-Delegation-Token"; @@ -256,6 +257,9 @@ public RMWebServices(final ResourceManager rm, Configuration conf) { this.filterAppsByUser = conf.getBoolean( YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + this.filterInvalidXMLChars = conf.getBoolean( + YarnConfiguration.FILTER_INVALID_XML_CHARS, + YarnConfiguration.DEFAULT_FILTER_INVALID_XML_CHARS); } RMWebServices(ResourceManager rm, Configuration conf, @@ -625,7 +629,8 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, } AppInfo app = new AppInfo(rm, rmapp, allowAccess, - WebAppUtils.getHttpSchemePrefix(conf), deSelectFields); + WebAppUtils.getHttpSchemePrefix(conf), deSelectFields, + filterInvalidXMLChars); allApps.add(app); } return allApps; @@ -957,7 +962,7 @@ public AppInfo getApp(@Context HttpServletRequest hsr, deSelectFields.initFields(unselectedFields); return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://", - deSelectFields); + deSelectFields, filterInvalidXMLChars); } @GET diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 63b6fe0..b2b3c1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -136,9 +136,15 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this(rm, app, hasAccess, schemePrefix, new DeSelectFields()); } - @SuppressWarnings({ "rawtypes", "unchecked" }) public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, String schemePrefix, DeSelectFields deSelects) { + this(rm, app, hasAccess, schemePrefix, deSelects, false); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, + String schemePrefix, DeSelectFields deSelects, + Boolean filterInvalidXMLChars) { this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); @@ -174,7 +180,12 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.diagnostics = app.getDiagnostics().toString(); if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; + } else { + if (filterInvalidXMLChars) { + this.diagnostics = stripInValidXMLCharacters(this.diagnostics); + } } + if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { this.applicationTags = Joiner.on(',').join(app.getApplicationTags()); @@ -346,6 +357,33 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, } } + /** + * This method ensures that the output String has only + * valid XML unicode characters as specified by the + * XML 1.0 standard. For reference, please see + * + * the standard. + * + * @param in The String whose non-valid characters we want to remove. + * @return The in String, stripped of non-valid characters. + */ + public static String stripInValidXMLCharacters(String in) { + StringBuffer out = new StringBuffer(); + char current; + + for (int i = 0; i < in.length(); i++) { + current = in.charAt(i); + if ((current == 0x9) || + (current == 0xA) || + (current == 0xD) || + ((current >= 0x20) && (current <= 0xD7FF)) || + ((current >= 0xE000) && (current <= 0xFFFD)) || + ((current >= 0x10000) && (current <= 0x10FFFF))) + out.append(current); + } + return out.toString(); + } + public boolean isTrackingUrlReady() { return !this.trackingUrlIsNotReady; } -- 2.7.4 (Apple Git-66)