From 4cb201753c28af32db7063e93d95a04a099b0c2d Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Tue, 14 Jan 2020 16:48:40 +0100 Subject: [PATCH] YARN-10029. Add option to UIv2 to get container logs from the new JHS API --- .../main/webapp/app/adapters/jhs-health.js | 30 ++++++ .../src/main/webapp/app/adapters/yarn-log.js | 1 + .../webapp/app/controllers/application.js | 1 + .../webapp/app/controllers/yarn-app/logs.js | 8 ++ .../main/webapp/app/initializers/loader.js | 95 +++++++++++-------- .../src/main/webapp/app/models/jhs-health.js | 27 ++++++ .../src/main/webapp/app/routes/application.js | 1 + .../main/webapp/app/routes/yarn-app/logs.js | 3 + .../src/main/webapp/app/services/hosts.js | 4 + .../webapp/app/templates/yarn-app/logs.hbs | 1 + .../src/main/webapp/config/default-config.js | 5 +- 11 files changed, 135 insertions(+), 41 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/jhs-health.js create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/jhs-health.js diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/jhs-health.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/jhs-health.js new file mode 100644 index 00000000000..b55ac0b271d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/jhs-health.js @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import RESTAbstractAdapter from './restabstract'; + +export default RESTAbstractAdapter.extend({ + address: "jhsAddress", + restNameSpace: "jhs", + serverName: "JHS", + + urlForQueryRecord(/*query, modelName*/) { + var url = this.buildURL(); + return url + '/info'; + } +}); \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-log.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-log.js index 66f34061546..cf8408550de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-log.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-log.js @@ -19,6 +19,7 @@ import AbstractAdapter from './abstract'; export default AbstractAdapter.extend({ + // TODO this should be configurable address: "timelineWebAddress", restNameSpace: "timelineV2Log", serverName: "ATS", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js index 34702aca2df..25540a9babe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js @@ -74,6 +74,7 @@ export default Ember.Controller.extend({ return null; }.property('model.userInfo'), + // TODO add JHS health isTimelineUnHealthy: function() { if (this.model && this.model.timelineHealth) { return this.model.timelineHealth.get('isTimelineUnHealthy'); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js index d7eac3ffd5b..16a185cbe3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js @@ -265,6 +265,14 @@ export default Ember.Controller.extend({ return logAggregationStatus !== "SUCCEEDED"; }), + isJHSAndTimelineUnhealthy: function() { + if (this.model && (this.model.jhsHealth || this.model.timelineHealth)) { + // TODO what should be checked here? + return true; + } + }.property('model.isJHSAndTimelineUnhealthy') + + // TODO add JHS healthiness isTimelineUnHealthy: function() { if (this.model && this.model.timelineHealth) { return this.model.timelineHealth.get('isTimelineUnHealthy'); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js index ad0132cd60f..c764d8037b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js @@ -20,31 +20,45 @@ import Ember from 'ember'; -function getYarnHttpProtocolScheme(rmhost, application) { +function getConfigFromYarn(rmhost, application, config) { var httpUrl = window.location.protocol + '//' + (ENV.hosts.localBaseAddress? ENV.hosts.localBaseAddress + '/' : '') + rmhost; - httpUrl += '/conf?name=yarn.http.policy'; - Ember.Logger.log("yarn.http.policy URL is: " + httpUrl); + httpUrl += '/conf?name=' + config; + Ember.Logger.log("The URL is: " + httpUrl); - var protocolScheme = ""; - $.ajax({ - type: 'GET', - dataType: 'json', - async: false, - context: this, - url: httpUrl, - success: function(data) { - protocolScheme = data.property.value; - Ember.Logger.log("Protocol scheme from RM: " + protocolScheme); + var configValue = ""; + $.ajax({ + type: 'GET', + dataType: 'json', + async: false, + context: this, + url: httpUrl, + success: function(data) { + configValue = data.property.value; + Ember.Logger.log("Value of the config returned from RM: " + configValue); - application.advanceReadiness(); - }, - error: function() { - application.advanceReadiness(); - } - }); - return protocolScheme; + application.advanceReadiness(); + }, + error: function() { + application.advanceReadiness(); + } + }); + return configValue; +} + +function getJHSURL(rmhost, application, isHttpsSchemeEnabled) { + var config = ''; + if (isHttpsSchemeEnabled) { + config = 'mapreduce.jobhistory.webapp.https.address'; + } else { + config = 'mapreduce.jobhistory.webapp.address'; + } + return getConfigFromYarn(rmhost, application, config); +} + +function getYarnHttpProtocolScheme(rmhost, application) { + return getConfigFromYarn(rmhost, application, 'yarn.http.policy'); } function getTimeLineURL(rmhost, isHttpsSchemeEnabled) { @@ -134,6 +148,19 @@ function getNodeManagerPort(rmhost, application) { return port; } +function transformURL(url, hostname) { + url = url.replace(/(^\w+:|^)\/\//, ''); + + var address = url.split(":")[0]; + var port = url.split(":")[1]; + if (address === "0.0.0.0" || address === "localhost") { + url = hostname + ":" + port; + } + + Ember.Logger.log("The transformed URL is: " + url); + return url; +} + function updateConfigs(application) { var hostname = window.location.hostname; var rmhost = hostname + (window.location.port ? ':' + window.location.port: '') + @@ -162,6 +189,12 @@ function updateConfigs(application) { Ember.Logger.log("NodeMananger port: " + nodeManagerPort); ENV.nodeManagerPort = nodeManagerPort; + if (!ENV.hosts.jhsAddress) { + var jhsAddress = getJHSURL(rmhost, application, isHttpsSchemeEnabled); + jhsAddress = transformURL(jhsAddress, hostname); + ENV.hosts.jhsAddress = jhsAddress; + } + if(!ENV.hosts.timelineWebAddress) { var timelinehost = ""; $.ajax({ @@ -172,19 +205,10 @@ function updateConfigs(application) { url: getTimeLineURL(rmhost, isHttpsSchemeEnabled), success: function(data) { timelinehost = data.property.value; - timelinehost = timelinehost.replace(/(^\w+:|^)\/\//, ''); + timelinehost = transformURL(timelinehost, hostname); ENV.hosts.timelineWebAddress = timelinehost; - - var address = timelinehost.split(":")[0]; - var port = timelinehost.split(":")[1]; - Ember.Logger.log("Timeline Address from RM: " + timelinehost); - if(address === "0.0.0.0" || address === "localhost") { - var updatedAddress = hostname + ":" + port; - ENV.hosts.timelineWebAddress = updatedAddress; - Ember.Logger.log("Timeline Updated Address: " + updatedAddress); - } application.advanceReadiness(); }, error: function() { @@ -206,19 +230,10 @@ function updateConfigs(application) { url: getTimeLineV1URL(rmhost, isHttpsSchemeEnabled), success: function(data) { timelinehost = data.property.value; - timelinehost = timelinehost.replace(/(^\w+:|^)\/\//, ''); + timelinehost = transformURL(timelinehost, hostname); ENV.hosts.timelineV1WebAddress = timelinehost; - - var address = timelinehost.split(":")[0]; - var port = timelinehost.split(":")[1]; - Ember.Logger.log("Timeline V1 Address from RM: " + timelinehost); - if(address === "0.0.0.0" || address === "localhost") { - var updatedAddress = hostname + ":" + port; - ENV.hosts.timelineV1WebAddress = updatedAddress; - Ember.Logger.log("Timeline V1 Updated Address: " + updatedAddress); - } application.advanceReadiness(); }, error: function() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/jhs-health.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/jhs-health.js new file mode 100644 index 00000000000..f487cde0be6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/jhs-health.js @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import DS from 'ember-data'; + +export default DS.Model.extend({ + healthStatus: DS.attr('string'), + + isJHSUnHealthy: function() { + return this.get('startedOn') !== null; + }.property('healthStatus') +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js index ead17e1ac53..46eb249ce74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js @@ -28,6 +28,7 @@ export default AbstractRoute.extend({ userInfo: this.store.findAll('cluster-user-info', {reload: true}).catch(function() { return null; }), + // TODO add JHS health timelineHealth: this.store.queryRecord('timeline-health', {}).catch(function() { return null; }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js index 9719f804f04..07d79d6ceaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js @@ -32,6 +32,9 @@ export default AbstractRoute.extend(AppAttemptMixin, { return []; }), app: this.fetchAppInfoFromRMorATS(app_id, this.store), + jhsHealth: this.store.queryRecord('jhs-health', {}).catch(function() { + return null; + }) timelineHealth: this.store.queryRecord('timeline-health', {}).catch(function() { return null; }) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/services/hosts.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/services/hosts.js index a53451b86f3..cda6e649c3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/services/hosts.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/services/hosts.js @@ -80,6 +80,10 @@ export default Ember.Service.extend({ return this.normalizeURL(this.get("env.app.hosts.rmWebAddress")); }), + jhsAddress: Ember.computed(function() { + return this.normalizeURL(this.get("env.get.hosts.jhsAddress")); + }), + dashWebAddress: Ember.computed(function () { return this.normalizeURL(this.get("env.app.hosts.dashWebAddress")); }), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs index b6dd318779c..66ee12dc2f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs @@ -32,6 +32,7 @@ Loading... {{/if}} + {{#if isTimelineUnHealthy}}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js index 9716488bb9d..5b1d72e4647 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js @@ -16,12 +16,14 @@ * limitations under the License. */ +// TODO maybe we have to add here a JHS url as well ? module.exports = { // YARN UI App configurations hosts: { localBaseAddress: "", timelineWebAddress: "localhost:8188", timelineV1WebAddress: "localhost:8188", rmWebAddress: "localhost:8088", + jhsAddress: "localhost:19888", protocolScheme: "http:", isSecurityEnabled: "" }, @@ -33,6 +35,7 @@ module.exports = { // YARN UI App configurations timelineV2: 'ws/v2/timeline', timelineV2Log: 'ws/v2/applicationlog', dashService: 'app/v1/services', - node: '{nodeAddress}/ws/v1/node' + node: '{nodeAddress}/ws/v1/node', + jhs: '/ws/v1/history' }, }; -- 2.21.0